bobpaulin commented on code in PR #11872:
URL: https://github.com/apache/nifi/pull/11872#discussion_r4606483513
##########
nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java:
##########
@@ -586,10 +618,155 @@ private void syncFromProviderForRead(final ConnectorNode
connector) {
}
@Override
- public Future<Void> startConnector(final ConnectorNode connector) {
+ public Future<Void> startConnector(final ConnectorNode connector, final
ConnectorUpdateContext persistenceContext) {
+ Objects.requireNonNull(persistenceContext, "Flow persistence context
is required");
+
+ final String connectorId = connector.getIdentifier();
+ final CompletableFuture<Void> runningEvolution =
evolveThenStartFuturesByConnectorId.get(connectorId);
+ if (runningEvolution != null) {
+ return runningEvolution;
+ }
+ if (!connector.isPendingFlowEvolution()) {
+ return connector.start(lifecycleExecutor);
+ }
+
+ final CompletableFuture<Void> evolutionFuture = new
CompletableFuture<>();
+ final CompletableFuture<Void> existingEvolution =
evolveThenStartFuturesByConnectorId.putIfAbsent(connectorId, evolutionFuture);
+ if (existingEvolution != null) {
+ return existingEvolution;
+ }
+
+ try {
+ connector.prepareFlowEvolution();
+ } catch (final Throwable failure) {
+ evolutionFuture.completeExceptionally(failure);
+ evolveThenStartFuturesByConnectorId.remove(connectorId,
evolutionFuture);
+ if (failure instanceof final Error error) {
+ throw error;
+ }
+ return evolutionFuture;
+ }
+
+ try {
+ flowEvolutionCoordinatorExecutor.submit(() ->
evolveThenStart(connector, evolutionFuture, persistenceContext));
+ } catch (final Throwable failure) {
+ try {
+ connector.abortUpdate(failure);
+ } catch (final Throwable cleanupFailure) {
+ failure.addSuppressed(cleanupFailure);
+ }
+ evolutionFuture.completeExceptionally(failure);
+ evolveThenStartFuturesByConnectorId.remove(connectorId,
evolutionFuture);
+ if (failure instanceof final Error error) {
+ throw error;
+ }
+ }
+
+ return evolutionFuture;
+ }
+
+ private void evolveThenStart(final ConnectorNode connector, final
CompletableFuture<Void> evolutionFuture, final ConnectorUpdateContext
persistenceContext) {
+ boolean startAttempted = false;
+ try {
+ final Future<Void> startFuture = runFlowEvolution(connector,
persistenceContext);
+ startAttempted = true;
+ // The evolution is complete once its start is issued. A later
START or STOP then acts on the connector directly.
+
evolveThenStartFuturesByConnectorId.remove(connector.getIdentifier(),
evolutionFuture);
+ startFuture.get();
+ evolutionFuture.complete(null);
+ } catch (final Throwable failure) {
+ if (failure instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+
+ if (startAttempted) {
+ logger.error("{} was evolved but did not start", connector,
failure);
+ } else {
+ logger.error("Flow evolution failed for {}", connector,
failure);
+ }
+
+ handleFlowEvolutionFailure(connector, evolutionFuture, failure,
startAttempted);
+
+ if (failure instanceof final Error error) {
+ throw error;
+ }
+ } finally {
+
evolveThenStartFuturesByConnectorId.remove(connector.getIdentifier(),
evolutionFuture);
+ }
+ }
+
+ /**
+ * Runs the evolve-drain phase of the evolution, then applies the upgraded
flow as a configuration update.
+ *
+ * @return the future for starting the evolved connector
+ */
+ private Future<Void> runFlowEvolution(final ConnectorNode connector, final
ConnectorUpdateContext persistenceContext) throws Throwable {
+ awaitFlowEvolutionDrain(connector,
connector.evolveFlow(flowEvolutionCoordinatorExecutor));
+
+ // No node applies the upgraded flow until every node that still has
the outgoing flow has drained it.
+ waitForFlowEvolutionStatus(connector);
+
+ // Every node has drained, so as at startup the preparation is local
and no cluster-wide wait for UPDATING is needed.
+ // A peer that has not yet observed the drain barrier may still be
DRAINING or preparing.
+ connector.beginEvolvedFlowUpdate();
+ connector.prepareForUpdate();
+ completeUpdate(connector, persistenceContext,
Set.of(ConnectorState.PREPARING_FOR_UPDATE, ConnectorState.UPDATING,
ConnectorState.DRAINING));
+
+ // The connector stayed desired RUNNING throughout the evolution, so
the flow saved after the apply already records it as running.
+ logger.info("{} has been successfully evolved; starting", connector);
return connector.start(lifecycleExecutor);
}
+ private void awaitFlowEvolutionDrain(final ConnectorNode connector, final
Future<Void> future) throws Throwable {
+ try {
+ future.get(flowEvolutionDrainTimeout.toNanos(),
TimeUnit.NANOSECONDS);
+ } catch (final TimeoutException e) {
+ final FlowUpdateException timedOut = new FlowUpdateException("Flow
evolution drain timed out for " + connector, e);
+ connector.abortUpdate(timedOut);
+ throw timedOut;
+ } catch (final InterruptedException e) {
+ connector.abortUpdate(e);
+ throw e;
+ } catch (final CancellationException e) {
+ throw new FlowUpdateException("Flow evolution was cancelled for "
+ connector, e);
+ } catch (final ExecutionException e) {
+ final Throwable cause = e.getCause() == null ? e : e.getCause();
+ if (cause instanceof final CancellationException cancelled) {
+ throw new FlowUpdateException("Flow evolution was cancelled
for " + connector, cancelled);
+ }
+ throw cause;
+ }
+ }
+
+ private static Duration canonicalFlowEvolutionDrainTimeout(final Duration
timeout) {
+ if (timeout == null || timeout.isZero() || timeout.isNegative()) {
+ return DEFAULT_FLOW_EVOLUTION_DRAIN_TIMEOUT;
+ }
+
+ return timeout;
+ }
+
+ private void handleFlowEvolutionFailure(final ConnectorNode connector,
final CompletableFuture<Void> evolutionFuture,
+ final Throwable failure, final boolean startAttempted) {
+ // A failed evolution is not saved; the persisted flow stays as it was
last saved until the connector is corrected.
+ // A failed start of the evolved connector is an ordinary failed
start, because the upgraded flow has already been saved.
+ if (!startAttempted) {
+ try {
+ // While DRAINING this fails the evolution. Once preparation
for the upgraded flow has begun it leaves UPDATE_FAILED.
+ connector.abortUpdate(failure);
+ if (connector.getCurrentState() == ConnectorState.STOPPING &&
connector instanceof final StandardConnectorNode standardConnector) {
Review Comment:
[Bob] Yeah that was not so great. Fixed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]