This is an automated email from the ASF dual-hosted git repository. markap14 pushed a commit to branch NIFI-15258 in repository https://gitbox.apache.org/repos/asf/nifi.git
commit e60e994d4ecc7d9bed842e2d8c1bb1bf47a2184b Author: Mark Payne <[email protected]> AuthorDate: Fri Dec 19 14:47:18 2025 -0500 NIFI-15367: Ensure that Connectors' implicit parameter contexts are n… (#10665) * NIFI-15367: Ensure that Connectors' implicit parameter contexts are not registered with ParameterContextManager; added system test to verify; fixed bug in ConnectorAssetsIT where it did not wait for connector to return to its 'STOPPED' state before attempting to delete * NIFI-15367: Bug fixes; review feedback --- .../nifi/controller/flow/AbstractFlowManager.java | 34 ++++++++++++++++++++-- .../apache/nifi/controller/flow/FlowManager.java | 25 +++++++++++----- .../connector/StandardConnectorRepository.java | 2 ++ .../components/connector/StandardFlowContext.java | 8 +++-- .../flow/FlowControllerFlowContextFactory.java | 12 ++++++-- .../nifi/controller/flow/StandardFlowManager.java | 4 +-- .../controller/flow/TestStandardFlowManager.java | 1 + .../web/dao/impl/StandardConnectorDAOTest.java | 11 +++++++ .../nifi/connectors/tests/system/NopConnector.java | 17 +++++++++-- .../apache/nifi/tests/system/NiFiClientUtil.java | 21 +++++++++++++ .../tests/system/connectors/ConnectorAssetsIT.java | 2 ++ .../tests/system/connectors/ConnectorCrudIT.java | 5 ++++ 12 files changed, 122 insertions(+), 20 deletions(-) diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java index a08000428c..a694c5be35 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java @@ -57,6 +57,7 @@ import org.apache.nifi.validation.RuleViolationsManager; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -628,8 +629,16 @@ public abstract class AbstractFlowManager implements FlowManager { @Override public ParameterContext createParameterContext(final String id, final String name, final String description, - final Map<String, Parameter> parameters, final List<String> inheritedContextIds, - final ParameterProviderConfiguration parameterProviderConfiguration) { + final Map<String, Parameter> parameters, final List<String> inheritedContextIds, + final ParameterProviderConfiguration parameterProviderConfiguration) { + + return createParameterContext(id, name, description, parameters, inheritedContextIds, parameterProviderConfiguration, true); + } + + protected ParameterContext createParameterContext(final String id, final String name, final String description, + final Map<String, Parameter> parameters, final List<String> inheritedContextIds, + final ParameterProviderConfiguration parameterProviderConfiguration, final boolean register) { + final boolean namingConflict = parameterContextManager.getParameterContexts().stream() .anyMatch(paramContext -> paramContext.getName().equals(name)); @@ -660,10 +669,29 @@ public abstract class AbstractFlowManager implements FlowManager { parameterContext.setInheritedParameterContexts(parameterContextList); } - parameterContextManager.addParameterContext(parameterContext); + if (register) { + parameterContextManager.addParameterContext(parameterContext); + } + return parameterContext; } + @Override + public ParameterContext duplicateParameterContext(final String id, final ParameterContext source) { + final Map<String, Parameter> parameterMap = new HashMap<>(); + for (final Parameter parameter : source.getParameters().values()) { + parameterMap.put(parameter.getDescriptor().getName(), parameter); + } + + final List<String> inheritedContextIds = new ArrayList<>(); + for (final ParameterContext inherited : source.getInheritedParameterContexts()) { + inheritedContextIds.add(inherited.getIdentifier()); + } + + return createParameterContext(id, source.getName(), source.getDescription(), + parameterMap, inheritedContextIds, source.getParameterProviderConfiguration(), false); + } + @Override public void withParameterContextResolution(final Runnable parameterContextAction) { withParameterContextResolution.set(true); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java index 8ef1ed6cff..266e9888d2 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java @@ -360,20 +360,29 @@ public interface FlowManager extends ParameterProviderLookup { * <code>IllegalStateException</code> is thrown. See {@link FlowManager#withParameterContextResolution(Runnable)} * for example usage. * - * @param id The unique id - * @param name The ParameterContext name - * @param description The ParameterContext description - * @param parameters The Parameters - * @param inheritedContextIds The identifiers of any Parameter Contexts that the newly created Parameter Context should inherit from. The order of the identifiers in the List determines the + * @param id the unique id + * @param name the ParameterContext name + * @param description the ParameterContext description + * @param parameters the Parameters + * @param inheritedContextIds the identifiers of any Parameter Contexts that the newly created Parameter Context should inherit from. The order of the identifiers in the List determines the * order in which parameters with conflicting names are resolved. I.e., the Parameter Context whose ID comes first in the List is preferred. - * @param parameterProviderConfiguration Optional configuration for a ParameterProvider - * @return The created ParameterContext - * @throws IllegalStateException If <code>parameterContexts</code> is not empty and this method is called without being wrapped + * @param parameterProviderConfiguration optional configuration for a ParameterProvider + * @return the created ParameterContext + * @throws IllegalStateException if <code>parameterContexts</code> is not empty and this method is called without being wrapped * by {@link FlowManager#withParameterContextResolution(Runnable)} */ ParameterContext createParameterContext(String id, String name, String description, Map<String, Parameter> parameters, List<String> inheritedContextIds, ParameterProviderConfiguration parameterProviderConfiguration); + /** + * Creates a duplicate of the given ParameterContext with the provided id. This does not register the Parameter Context + * with the ParameterContextManager. + * @param id the id of the new ParameterContext + * @param source the ParameterContext to duplicate + * @return the duplicated ParameterContext + */ + ParameterContext duplicateParameterContext(String id, ParameterContext source); + /** * Performs the given ParameterContext-related action, and then resolves all inherited ParameterContext references. * Example usage: <br/><br/> diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java index dfa7a91a94..34fe990aa1 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java @@ -147,8 +147,10 @@ public class StandardConnectorRepository implements ConnectorRepository { // If the initial desired state was RUNNING, start the connector again. Otherwise, stop it. // We don't simply leave it be as the prepareForUpdate / update may have changed the state of some components. if (initialDesiredState == ConnectorState.RUNNING) { + logger.info("Connector {} has been successfully updated; starting Connector to resume initial state", connector); connector.start(lifecycleExecutor); } else { + logger.info("Connector {} has been successfully updated; stopping Connector to resume initial state", connector); connector.stop(lifecycleExecutor); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFlowContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFlowContext.java index ea855e6215..16ebbc3e39 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFlowContext.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardFlowContext.java @@ -27,6 +27,8 @@ import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.logging.ComponentLog; +import java.util.Collections; + public class StandardFlowContext implements FrameworkFlowContext { private final ProcessGroup managedProcessGroup; private final MutableConnectorConfigurationContext configurationContext; @@ -83,7 +85,10 @@ public class StandardFlowContext implements FrameworkFlowContext { throw new FlowUpdateException("Flow is not in a state that allows the requested updated", e); } - managedProcessGroup.updateFlow(versionedExternalFlow, managedProcessGroup.getIdentifier(), false, true, true); + final VersionedExternalFlow withoutParameterContext = new VersionedExternalFlow(); + withoutParameterContext.setFlowContents(versionedExternalFlow.getFlowContents()); + withoutParameterContext.setParameterContexts(Collections.emptyMap()); + managedProcessGroup.updateFlow(withoutParameterContext, managedProcessGroup.getIdentifier(), false, true, true); final ConnectorParameterLookup parameterLookup = new ConnectorParameterLookup(versionedExternalFlow.getParameterContexts().values(), assetManager); getParameterContext().updateParameters(parameterLookup.getParameterValues()); @@ -101,7 +106,6 @@ public class StandardFlowContext implements FrameworkFlowContext { } } - @Override public Bundle getBundle() { return bundle; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/FlowControllerFlowContextFactory.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/FlowControllerFlowContextFactory.java index c488caf03d..b7ed11d323 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/FlowControllerFlowContextFactory.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/FlowControllerFlowContextFactory.java @@ -29,9 +29,9 @@ import org.apache.nifi.components.connector.facades.standalone.StandaloneParamet import org.apache.nifi.controller.FlowController; import org.apache.nifi.flow.Bundle; import org.apache.nifi.flow.VersionedExternalFlow; -import org.apache.nifi.flow.VersionedParameterContext; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.ParameterContext; import org.apache.nifi.registry.flow.mapping.ComponentIdLookup; import org.apache.nifi.registry.flow.mapping.FlowMappingOptions; import org.apache.nifi.registry.flow.mapping.InstantiatedVersionedProcessGroup; @@ -96,15 +96,21 @@ public class FlowControllerFlowContextFactory implements FlowContextFactory { .build(); final VersionedComponentFlowMapper flowMapper = new VersionedComponentFlowMapper(flowController.getExtensionManager(), flowMappingOptions); - final Map<String, VersionedParameterContext> parameterContexts = flowMapper.mapParameterContexts(sourceGroup, true, Map.of()); final InstantiatedVersionedProcessGroup versionedGroup = flowMapper.mapProcessGroup(sourceGroup, flowController.getControllerServiceProvider(), flowController.getFlowManager(), true); final VersionedExternalFlow versionedExternalFlow = new VersionedExternalFlow(); versionedExternalFlow.setFlowContents(versionedGroup); versionedExternalFlow.setExternalControllerServices(Map.of()); versionedExternalFlow.setParameterProviders(Map.of()); - versionedExternalFlow.setParameterContexts(parameterContexts); + versionedExternalFlow.setParameterContexts(Map.of()); destinationGroup.updateFlow(versionedExternalFlow, componentIdSeed, false, true, true); + + final String duplicateContextId = UUID.nameUUIDFromBytes((destinationGroup.getIdentifier() + "-param-context").getBytes(StandardCharsets.UTF_8)).toString(); + final ParameterContext sourceContext = sourceGroup.getParameterContext(); + if (sourceContext != null) { + final ParameterContext duplicateParameterContext = flowController.getFlowManager().duplicateParameterContext(duplicateContextId, sourceContext); + destinationGroup.setParameterContext(duplicateParameterContext); + } } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java index c48288bd71..7cf4bc8707 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java @@ -757,9 +757,9 @@ public class StandardFlowManager extends AbstractFlowManager implements FlowMana final String paramContextId = UUID.nameUUIDFromBytes((id + "-parameter-context").getBytes(StandardCharsets.UTF_8)).toString(); final String paramContextName = "Connector " + id + " Parameter Context"; + final String parameterContextDescription = "Implicit Parameter Context for Connector " + id; final ParameterContext managedParameterContext = createParameterContext(paramContextId, paramContextName, - "Implicit Parameter Context for Connector " + id, - Collections.emptyMap(), Collections.emptyList(), null); + parameterContextDescription, Collections.emptyMap(), Collections.emptyList(), null, false); managedRootGroup.setParameterContext(managedParameterContext); final ConnectorRepository connectorRepository = flowController.getConnectorRepository(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/TestStandardFlowManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/TestStandardFlowManager.java index 9f11f77c60..33ec45aac0 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/TestStandardFlowManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/flow/TestStandardFlowManager.java @@ -95,6 +95,7 @@ public class TestStandardFlowManager { when(bundle.getClassLoader()).thenReturn(NopConnector.class.getClassLoader()); flowManager = new StandardFlowManager(nifiProperties, sslContext, flowController, flowFileEventRepository, parameterContextManager); + when(flowController.getFlowManager()).thenReturn(flowManager); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java index f3c480efd5..3f13d8f569 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardConnectorDAOTest.java @@ -33,6 +33,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; import java.util.Collections; import java.util.List; @@ -41,11 +43,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) class StandardConnectorDAOTest { private StandardConnectorDAO connectorDAO; @@ -84,6 +88,13 @@ class StandardConnectorDAOTest { connectorDAO.setFlowController(flowController); when(flowController.getConnectorRepository()).thenReturn(connectorRepository); + when(connectorRepository.getAssetRepository()).thenReturn(connectorAssetRepository); + + final MutableConnectorConfigurationContext configContext = mock(MutableConnectorConfigurationContext.class); + when(configContext.toConnectorConfiguration()).thenReturn(mock(ConnectorConfiguration.class)); + final FrameworkFlowContext activeContext = mock(FrameworkFlowContext.class); + when(activeContext.getConfigurationContext()).thenReturn(configContext); + when(connectorNode.getActiveFlowContext()).thenReturn(activeContext); } @Test diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java index 702a4574ad..3d28240ca2 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/connectors/tests/system/NopConnector.java @@ -29,12 +29,14 @@ import org.apache.nifi.components.connector.ConnectorPropertyGroup; import org.apache.nifi.components.connector.PropertyType; import org.apache.nifi.components.connector.components.FlowContext; import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedParameter; +import org.apache.nifi.flow.VersionedParameterContext; import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.processor.util.StandardValidators; -import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; public class NopConnector extends AbstractConnector { @@ -81,8 +83,19 @@ public class NopConnector extends AbstractConnector { final VersionedProcessGroup group = new VersionedProcessGroup(); group.setName("Nop Flow"); + final VersionedParameter someText = new VersionedParameter(); + someText.setName("SOME_TEXT"); + someText.setValue("Lorem ipsum"); + someText.setSensitive(false); + someText.setProvided(false); + someText.setReferencedAssets(List.of()); + + final VersionedParameterContext parameterContext = new VersionedParameterContext(); + parameterContext.setName("Nop Parameter Context"); + parameterContext.setParameters(Set.of(someText)); + final VersionedExternalFlow flow = new VersionedExternalFlow(); - flow.setParameterContexts(Collections.emptyMap()); + flow.setParameterContexts(Map.of(parameterContext.getName(), parameterContext)); flow.setFlowContents(group); return flow; } diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java index 28b43e150c..7a16c754c9 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java @@ -397,6 +397,27 @@ public class NiFiClientUtil { } } + public void waitForConnectorStopped(final String connectorId) throws NiFiClientException, IOException, InterruptedException { + waitForConnectorState(connectorId, ConnectorState.STOPPED); + } + + public void waitForConnectorState(final String connectorId, final ConnectorState desiredState) throws InterruptedException, NiFiClientException, IOException { + int iteration = 0; + while (true) { + final ConnectorEntity entity = getConnectorClient().getConnector(connectorId); + final String state = entity.getComponent().getState(); + if (desiredState.name().equals(state)) { + return; + } + + if (iteration++ % 30 == 0) { // Every 3 seconds log status + logger.info("Connector with ID {} has state {} but waiting for state {}.", connectorId, state, desiredState); + } + + Thread.sleep(100L); + } + } + public ParameterProviderEntity createParameterProvider(final String simpleTypeName) throws NiFiClientException, IOException { return createParameterProvider(NiFiSystemIT.TEST_PARAM_PROVIDERS_PACKAGE + "." + simpleTypeName, NiFiSystemIT.NIFI_GROUP_ID, NiFiSystemIT.TEST_EXTENSIONS_ARTIFACT_ID, nifiVersion); } diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorAssetsIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorAssetsIT.java index 59b46ddd82..3a83731776 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorAssetsIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorAssetsIT.java @@ -195,6 +195,8 @@ public class ConnectorAssetsIT extends NiFiSystemIT { assertFalse(assetStillPresent); + // Wait for Connector to stop before attempting to delete it. + getClientUtil().waitForConnectorStopped(connectorAfterApply.getId()); connectorClient.deleteConnector(connectorAfterApply); } } diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorCrudIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorCrudIT.java index 0179c17496..085177dad4 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorCrudIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/ConnectorCrudIT.java @@ -24,6 +24,7 @@ import org.apache.nifi.web.api.dto.ConfigVerificationResultDTO; import org.apache.nifi.web.api.dto.ConnectorConfigurationDTO; import org.apache.nifi.web.api.dto.ConnectorValueReferenceDTO; import org.apache.nifi.web.api.entity.ConnectorEntity; +import org.apache.nifi.web.api.entity.ParameterContextsEntity; import org.apache.nifi.web.api.entity.ParameterProviderEntity; import org.junit.jupiter.api.Test; @@ -43,6 +44,10 @@ public class ConnectorCrudIT extends NiFiSystemIT { final ConnectorEntity connector = getClientUtil().createConnector("NopConnector"); assertNotNull(connector); + // Ensure that Parameter Context is not created for the Connector + final ParameterContextsEntity contextsEntity = getNifiClient().getParamContextClient().getParamContexts(); + assertEquals(0, contextsEntity.getParameterContexts().size()); + // Configure the connector and apply the configuration getClientUtil().configureConnector(connector, "Ignored Step", Map.of("Ignored Property", "Hello, World!")); getClientUtil().applyConnectorUpdate(connector);
