bobpaulin commented on code in PR #11872:
URL: https://github.com/apache/nifi/pull/11872#discussion_r4552816308


##########
nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java:
##########
@@ -2039,6 +2123,329 @@ public void 
testInheritingConfigurationKeepsTransitivelyGatedRequiredPropertyIrr
         connectorNode.verifyCanStart();
     }
 
+
+    @Test
+    public void testEvolveFlowReturnsSameTaskWhileHookIsPending() throws 
Exception {
+        final FlowEvolutionHookConnector connector = new 
FlowEvolutionHookConnector();
+        connector.leaveFutureIncomplete = true;
+        final StandardConnectorNode connectorNode = 
createConnectorNode(connector);
+        connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"), 
null, Map.of(), outgoingSnapshotBundle());
+
+        connectorNode.prepareFlowEvolution();
+        final Future<Void> evolution = 
connectorNode.evolveFlow(flowEvolutionExecutor);
+        assertTrue(connector.evolutionFuturePublished.await(5, 
TimeUnit.SECONDS));
+
+        assertSame(evolution, connectorNode.evolveFlow(flowEvolutionExecutor));
+        assertEquals(FlowEvolutionStatus.PHASE_ONE_PENDING, 
connectorNode.getFlowEvolutionStatus());
+        // The hook stays blocked, so the task has to be interrupted before 
the executor is closed.
+        evolution.cancel(true);
+    }
+
+    enum HookCancel {
+        STOP,
+        CANCEL_DRAIN,
+        ABORT
+    }
+
+    @ParameterizedTest
+    @EnumSource(HookCancel.class)
+    public void testCancelsBlockedFlowEvolutionHook(final HookCancel action) 
throws Exception {
+        assertHookCancelSetsDesiredStopped(switch (action) {
+            case STOP -> node -> node.stop(scheduler);
+            case CANCEL_DRAIN -> StandardConnectorNode::cancelDrainFlowFiles;
+            case ABORT -> node -> node.abortUpdate(new 
FlowUpdateException("cancel"));
+        });
+    }
+
+    @Test
+    public void testEvolveFlowAfterAbortDoesNotInvokeHook() throws Exception {
+        final FlowEvolutionHookConnector connector = new 
FlowEvolutionHookConnector();
+        final StandardConnectorNode connectorNode = 
createConnectorNode(connector);
+        connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"), 
null, Map.of(), outgoingSnapshotBundle());
+
+        connectorNode.prepareFlowEvolution();
+        connectorNode.abortUpdate(new FlowUpdateException("cancel"));
+
+        assertThrows(FlowUpdateException.class, () -> 
connectorNode.evolveFlow(flowEvolutionExecutor));
+        assertEquals(0, connector.hookCalls);
+    }
+
+    @Test
+    public void testResumeSkipsHook() throws Exception {
+        final FlowEvolutionHookConnector connector = new 
FlowEvolutionHookConnector();
+        final StandardConnectorNode connectorNode = 
createConnectorNode(connector);
+        connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"), 
null, Map.of(), outgoingSnapshotBundle());
+
+        connectorNode.prepareFlowEvolution();
+        connectorNode.evolveFlow(flowEvolutionExecutor).get(5, 
TimeUnit.SECONDS);
+        connectorNode.abortUpdate(new FlowUpdateException("persist failed"));
+        connectorNode.prepareFlowEvolution();
+
+        final Future<Void> resumed = 
connectorNode.evolveFlow(flowEvolutionExecutor);
+        resumed.get(5, TimeUnit.SECONDS);
+        assertEquals(1, connector.hookCalls);
+        assertEquals(FlowEvolutionStatus.PHASE_ONE_COMPLETE, 
connectorNode.getFlowEvolutionStatus());
+    }
+
+    @Test
+    public void testStaleEvolutionTaskCannotWriteIntoLaterEvolution() throws 
Exception {
+        final GatedActiveFlowConnector connector = new 
GatedActiveFlowConnector();
+        final StandardConnectorNode connectorNode = 
createConnectorNode(connector);
+        connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"), 
null, Map.of(), outgoingSnapshotBundle());
+        connectorNode.prepareFlowEvolution();
+        final Future<Void> evolution = 
connectorNode.evolveFlow(flowEvolutionExecutor);
+        assertTrue(connector.reachedActiveFlow.await(5, TimeUnit.SECONDS));
+
+        connectorNode.abortUpdate(new FlowUpdateException("cancel"));
+        connectorNode.prepareFlowEvolution();
+        connector.releaseActiveFlow.countDown();
+
+        assertThrows(CancellationException.class, () -> evolution.get(5, 
TimeUnit.SECONDS));
+        assertEquals(FlowEvolutionStatus.PHASE_ONE_PENDING, 
connectorNode.getFlowEvolutionStatus());
+        assertEquals(ConnectorState.DRAINING, connectorNode.getCurrentState());
+        
assertTrue(connectorNode.getValidationErrors().stream().noneMatch(result -> 
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject())));
+    }
+
+    private void assertHookCancelSetsDesiredStopped(final 
Consumer<StandardConnectorNode> cancel) throws Exception {
+        final FlowEvolutionHookConnector connector = new 
FlowEvolutionHookConnector();
+        connector.leaveFutureIncomplete = true;
+        final StandardConnectorNode connectorNode = 
createConnectorNode(connector);
+        connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"), 
null, Map.of(), outgoingSnapshotBundle());
+
+        connectorNode.prepareFlowEvolution();
+        final Future<Void> evolution = 
connectorNode.evolveFlow(flowEvolutionExecutor);
+        assertTrue(connector.hookEntered.await(5, TimeUnit.SECONDS));
+        assertTrue(connector.evolutionFuturePublished.await(5, 
TimeUnit.SECONDS));
+
+        cancel.accept(connectorNode);
+        assertEquals(ConnectorState.STOPPED, connectorNode.getDesiredState());
+        assertThrows(CancellationException.class, () -> evolution.get(2, 
TimeUnit.SECONDS));
+        assertTrue(connector.evolutionFuture.isCancelled());
+        assertEquals(FlowEvolutionStatus.FAILED, 
connectorNode.getFlowEvolutionStatus());
+        assertTrue(connectorNode.isPendingFlowEvolution());
+        assertEvolutionContextClosed(connector.context);
+    }
+
+    @Test
+    public void 
testFlowEvolutionFailureIsIndependentOfValidationStateUntilNextAttempt() throws 
Exception {
+        final ConnectorValidationTrigger validationTrigger = 
mock(ConnectorValidationTrigger.class);
+        final StandardConnectorNode connectorNode = createConnectorNode(new 
FlowEvolutionHookConnector(), validationTrigger);
+        connectorNode.markInvalid("Existing Failure", "existing failure");
+
+        connectorNode.failFlowEvolution("first evolution failure", null);
+        assertEquals(FlowEvolutionStatus.FAILED, 
connectorNode.getFlowEvolutionStatus());
+        assertEquals(2, connectorNode.getValidationErrors().size());
+        assertEquals(1, connectorNode.getValidationErrors().stream()
+                .filter(result -> 
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject()))
+                .count());
+
+        connectorNode.performValidation();
+        assertEquals(1, connectorNode.getValidationErrors().stream()
+                .filter(result -> 
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject()))
+                .count());
+        connectorNode.resetValidationState();
+        assertEquals(1, connectorNode.getValidationErrors().stream()
+                .filter(result -> 
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject()))
+                .count());
+
+        connectorNode.markInvalid("Replacement Failure", "replacement 
failure");
+        connectorNode.failFlowEvolution("duplicate evolution failure", null);
+        assertEquals(2, connectorNode.getValidationErrors().size());
+        final ValidationResult evolutionFailure = 
connectorNode.getValidationErrors().stream()
+                .filter(result -> 
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject()))
+                .findFirst()
+                .orElseThrow();
+        assertEquals("first evolution failure", 
evolutionFailure.getExplanation());
+        assertEquals(1, ((MockComponentLog) 
connectorNode.getComponentLog()).getErrorMessages().stream()
+                .filter(message -> message.getMsg().contains("Flow evolution 
failed"))
+                .count());
+
+        connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"), 
null, Map.of(), outgoingSnapshotBundle());
+        connectorNode.prepareFlowEvolution();
+
+        assertEquals(FlowEvolutionStatus.PHASE_ONE_PENDING, 
connectorNode.getFlowEvolutionStatus());
+        assertTrue(connectorNode.getValidationErrors().stream()
+                .noneMatch(result -> 
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject())));
+        assertTrue(connectorNode.getValidationErrors().stream()
+                .anyMatch(result -> "Replacement 
Failure".equals(result.getSubject())));
+    }
+
+    @Test
+    public void testEvolveFlowRejectsMissingManagedComponents() throws 
Exception {
+        final FlowEvolutionHookConnector connector = new 
FlowEvolutionHookConnector();
+        final ProcessorNode missingProcessor = mock(ProcessorNode.class);
+        when(missingProcessor.isExtensionMissing()).thenReturn(true);
+        
when(missingProcessor.getCanonicalClassName()).thenReturn("org.example.MissingProcessor");
+        when(missingProcessor.getBundleCoordinate()).thenReturn(new 
BundleCoordinate("org.example", "missing-processor", "1.0.0"));
+        
when(managedProcessGroup.findAllProcessors()).thenReturn(List.of(missingProcessor));
+        final StandardConnectorNode processorNode = 
createConnectorNode(connector);
+        processorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"), 
null, Map.of(), outgoingSnapshotBundle());
+
+        processorNode.prepareFlowEvolution();
+        final ExecutionException processorFailure = 
assertThrows(ExecutionException.class,
+                () -> processorNode.evolveFlow(flowEvolutionExecutor).get(5, 
TimeUnit.SECONDS));
+        
assertTrue(processorFailure.getCause().getMessage().contains("MissingProcessor"),
 processorFailure.getCause().getMessage());
+        assertEquals(FlowEvolutionStatus.FAILED, 
processorNode.getFlowEvolutionStatus());
+        assertEquals(ConnectorState.STOPPED, processorNode.getCurrentState());
+        assertEquals(0, connector.hookCalls);
+    }
+
+    @Test
+    public void 
testPendingEvolutionBlocksEndTroubleshootingWithQueuedFlowFiles() throws 
Exception {
+        final StandardConnectorNode connectorNode = createConnectorNode(new 
FlowEvolutionHookConnector());
+        connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"), 
null, Map.of(), outgoingSnapshotBundle());
+        connectorNode.restoreTroubleshootingState();
+        when(managedProcessGroup.getQueueSize()).thenReturn(new QueueSize(3, 
30L));
+
+        final IllegalStateException failure = 
assertThrows(IllegalStateException.class, 
connectorNode::verifyCanEndTroubleshooting);
+
+        assertTrue(failure.getMessage().contains("3 FlowFiles"), 
failure.getMessage());
+        assertTrue(failure.getMessage().contains("upgrade is pending"), 
failure.getMessage());
+    }
+
+    @Test
+    public void testFlowEvolutionRetriesAfterMissingSecretRestored() throws 
Exception {
+        final SecretReference secretReference = new 
SecretReference("provider-id", "Provider", "password", 
"Provider.group.password");
+        final AtomicReference<Secret> currentSecret = new AtomicReference<>();
+        when(secretsManager.getSecrets(anySet(), 
eq(false))).thenAnswer(invocation -> {
+            final Secret secret = currentSecret.get();
+            return secret == null ? Map.of() : Map.of(secretReference, secret);
+        });
+
+        final StandardConnectorNode connectorNode = createConnectorNode(new 
FlowEvolutionHookConnector(), secretsManager);
+        connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"),
+            strippedParameterContext("Source Database Password"), 
Map.of("Source Database Password", secretReference),
+            outgoingSnapshotBundle());
+
+        connectorNode.prepareFlowEvolution();
+        final ExecutionException failure = 
assertThrows(ExecutionException.class,
+                () -> connectorNode.evolveFlow(flowEvolutionExecutor).get(5, 
TimeUnit.SECONDS));
+        
assertTrue(failure.getCause().getMessage().contains("Provider.group.password"), 
failure.getCause().getMessage());
+        assertTrue(failure.getCause().getMessage().contains("Source Database 
Password"), failure.getCause().getMessage());
+        assertEquals(FlowEvolutionStatus.FAILED, 
connectorNode.getFlowEvolutionStatus());
+        assertEquals(ConnectorState.STOPPED, connectorNode.getCurrentState());
+
+        final Secret restoredSecret = mock(Secret.class);
+        when(restoredSecret.getValue()).thenReturn("current-password");
+        currentSecret.set(restoredSecret);
+        connectorNode.prepareFlowEvolution();
+        assertEquals(FlowEvolutionStatus.PHASE_ONE_PENDING, 
connectorNode.getFlowEvolutionStatus());
+        assertTrue(connectorNode.getValidationErrors().stream()
+            .noneMatch(result -> 
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject())));
+        connectorNode.evolveFlow(flowEvolutionExecutor).get(5, 
TimeUnit.SECONDS);
+
+        final ArgumentCaptor<Collection<ParameterValue>> valuesCaptor = 
ArgumentCaptor.forClass(Collection.class);
+        
verify(parameterContextFacade).updateParameters(valuesCaptor.capture());
+        assertEquals("current-password", 
valuesCaptor.getValue().iterator().next().getValue());
+        assertEquals(FlowEvolutionStatus.PHASE_ONE_COMPLETE, 
connectorNode.getFlowEvolutionStatus());
+    }
+
+    @Test
+    public void 
testCaptureAuthoritativeSnapshotPersistsStructuredParameterSecretReference() 
throws Exception {
+        final String parameterName = "Source Database Password";
+        final String secretValue = "current-password";
+        final SecretReference secretReference = new 
SecretReference("provider-id", "Provider", "password", 
"Provider.group.password");
+        final Secret secret = mock(Secret.class);
+        when(secret.getValue()).thenReturn(secretValue);
+        
when(secretsManager.getSecrets(anySet())).thenReturn(Map.of(secretReference, 
secret));
+
+        final Parameter parameter = new Parameter.Builder()
+            .name(parameterName)
+            .value(secretValue)
+            .sensitive(true)
+            .build();
+        final ParameterContext parameterContext = mock(ParameterContext.class);
+        
when(parameterContext.getIdentifier()).thenReturn("managed-parameters");
+        when(parameterContext.getName()).thenReturn("Managed Parameters");
+        
when(parameterContext.getInheritedParameterContextNames()).thenReturn(List.of());
+        
when(parameterContext.getParameters()).thenReturn(Map.of(parameter.getDescriptor(),
 parameter));
+
+        final StandardConnectorNode connectorNode = createConnectorNode(new 
FlowEvolutionHookConnector(), secretsManager);
+        
when(managedProcessGroup.getParameterContext()).thenReturn(parameterContext);
+        seedActiveConfiguration(connectorNode, REQUIRED_STEP, 
Map.of(REQUIRED_SECRET, secretReference));
+
+        connectorNode.captureAuthoritativeSnapshot();
+
+        final ConnectorFlowSnapshot snapshot = 
connectorNode.getAuthoritativeFlowSnapshot();
+        final VersionedParameter persistedParameter = 
snapshot.parameterContext().getParameters().iterator().next();
+        assertFalse(persistedParameter.isProvided());
+        assertNull(persistedParameter.getValue());
+        assertEquals(Map.of(parameterName, secretReference), 
snapshot.parameterSecretReferences());
+        assertEquals(connectorNode.getBundleCoordinate().getVersion(), 
snapshot.bundle().getVersion());
+        assertEquals(secretReference, 
snapshot.activeConfiguration().getNamedStepConfiguration(REQUIRED_STEP)
+            .configuration().getPropertyValues().get(REQUIRED_SECRET));
+    }
+
+    @Test
+    public void 
testRestorePersistedFlowKeepsComponentsStoppedWhenSecretMissing() throws 
Exception {
+        final SecretReference secretReference = new 
SecretReference("provider-id", "Provider", "password", 
"Provider.group.password");
+        when(secretsManager.getSecrets(anySet(), 
eq(false))).thenReturn(Map.of());
+        final StandardConnectorNode connectorNode = createConnectorNode(new 
FlowEvolutionHookConnector(), secretsManager);
+        clearInvocations(managedProcessGroup);
+
+        connectorNode.restorePersistedFlow(flowGroup("pre-upgrade"),
+            strippedParameterContext("Source Database Password"), 
Map.of("Source Database Password", secretReference), true);
+
+        
verify(managedProcessGroup).restoreFlowPreservingIdentifiers(any(VersionedExternalFlow.class),
 eq(false));
+    }
+
+    @Test
+    public void 
testRestorePersistedFlowUsesConfiguredAutoResumePolicyWhenSecretsResolved() 
throws Exception {
+        final StandardConnectorNode connectorNode = createConnectorNode(new 
FlowEvolutionHookConnector(), secretsManager);
+        clearInvocations(managedProcessGroup);
+
+        connectorNode.restorePersistedFlow(flowGroup("pre-upgrade"), null, 
Map.of(), true);
+
+        
verify(managedProcessGroup).restoreFlowPreservingIdentifiers(any(VersionedExternalFlow.class));
+        verify(managedProcessGroup, 
never()).restoreFlowPreservingIdentifiers(any(VersionedExternalFlow.class), 
anyBoolean());
+    }
+
+    @Test
+    public void 
testPreUpgradeFlowContextUsesSnapshotBundleAndRejectsFlowUpdate() throws 
Exception {

Review Comment:
   Removed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to