This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new d250631502bc CAMEL-27089: camel-core - parallel Multicast, Split,
Recipient List: fail the exchange when the route is removed during a part
(#30305)
d250631502bc is described below
commit d250631502bc80946fd42a65191a805ab62ccda8
Author: allthingssecurity <[email protected]>
AuthorDate: Wed Nov 18 17:50:58 2026 +0530
CAMEL-27089: camel-core - parallel Multicast, Split, Recipient List: fail
the exchange when the route is removed during a part (#30305)
A parallel Multicast, Split or Recipient List with its own thread pool runs
its parts as tasks on that
pool. When the route is stopped after the shutdown timeout and then removed
(or reloaded) while an
exchange still waits for a part, doShutdown shuts the pool down with
shutdownNow: the queued parts are
dropped, so the exchange is never completed, its caller waits for ever and
it stays in flight.
The processor now keeps its parallel tasks that are not done, and after
shutting down its own thread
pool fails each of them with a RouteStoppingException through the same done
flag as the other ways a
task completes, so the callback is called once.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../apache/camel/processor/MulticastProcessor.java | 44 ++++-
.../org/apache/camel/processor/RecipientList.java | 6 +-
.../ParallelProcessingRemoveRouteTest.java | 199 +++++++++++++++++++++
3 files changed, 240 insertions(+), 9 deletions(-)
diff --git
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java
index ea01a7a57531..b3889de76cbd 100644
---
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java
+++
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java
@@ -23,6 +23,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
@@ -49,6 +50,7 @@ import org.apache.camel.Navigate;
import org.apache.camel.Processor;
import org.apache.camel.Producer;
import org.apache.camel.Route;
+import org.apache.camel.RouteStoppingException;
import org.apache.camel.RuntimeCamelException;
import org.apache.camel.StreamCache;
import org.apache.camel.Traceable;
@@ -168,6 +170,9 @@ public class MulticastProcessor extends BaseProcessorSupport
protected final Processor onPrepare;
protected final ProcessorExchangeFactory processorExchangeFactory;
private final Lock lock = new ReentrantLock();
+ // the parallel tasks that are not done yet, so doShutdown can fail those
whose parts the shut down thread pool
+ // never runs
+ private final Set<MulticastTask> parallelTasks =
ConcurrentHashMap.newKeySet();
private final AsyncProcessorAwaitManager awaitManager;
private final CamelContext camelContext;
private final InternalProcessorFactory internalProcessorFactory;
@@ -405,6 +410,12 @@ public class MulticastProcessor extends
BaseProcessorSupport
} catch (RejectedExecutionException e) {
state.reject();
}
+ if (shutdownExecutorService && executorService.isShutdown()) {
+ // doShutdown shut the thread pool down meanwhile, maybe
before it saw this task: with the CallerRuns
+ // policy the pool drops the task without an exception, so
fail the exchange here (a no-op when
+ // doShutdown or reject() completed it already)
+ state.doFailed(new RouteStoppingException("The exchange cannot
complete: the thread pool is shut down"));
+ }
} else {
if (exchange.isTransacted()) {
reactiveExecutor.scheduleQueue(state);
@@ -490,6 +501,20 @@ public class MulticastProcessor extends
BaseProcessorSupport
} else {
this.completion = new AsyncCompletionService<>(transacted ?
txScheduler : scheduler, !isStreaming(), lock);
}
+ if (isParallelProcessing()) {
+ parallelTasks.add(this);
+ }
+ }
+
+ /**
+ * Marks this task as done, which only one caller can do.
+ */
+ private boolean markDone() {
+ if (done.compareAndSet(false, true)) {
+ parallelTasks.remove(this);
+ return true;
+ }
+ return false;
}
@Override
@@ -600,13 +625,13 @@ public class MulticastProcessor extends
BaseProcessorSupport
}
protected void doTimeoutDone(Exchange exchange, boolean forceExhaust) {
- if (done.compareAndSet(false, true)) {
+ if (markDone()) {
MulticastProcessor.this.doDone(original, exchange, pairs,
callback, false, forceExhaust);
}
}
protected void doDone(Exchange exchange, boolean forceExhaust) {
- if (done.compareAndSet(false, true)) {
+ if (markDone()) {
cancelTimeoutTask();
MulticastProcessor.this.doDone(original, exchange, pairs,
callback, false, forceExhaust);
}
@@ -626,7 +651,7 @@ public class MulticastProcessor extends BaseProcessorSupport
* {@link VirtualMachineError} (set on the exchange wrapped in a
CamelExecutionException).
*/
protected void doFailed(Throwable e) {
- if (done.compareAndSet(false, true)) {
+ if (markDone()) {
cancelTimeoutTask();
original.setException(e);
MulticastProcessor.this.doDone(original, null, pairs,
callback, false, false);
@@ -646,7 +671,7 @@ public class MulticastProcessor extends BaseProcessorSupport
* Same as {@link #doTimeoutFailed(Exception)} for any {@link
Throwable}.
*/
protected void doTimeoutFailed(Throwable e) {
- if (done.compareAndSet(false, true)) {
+ if (markDone()) {
original.setException(e);
MulticastProcessor.this.doDone(original, null, pairs,
callback, false, false);
} else {
@@ -1485,6 +1510,14 @@ public class MulticastProcessor extends
BaseProcessorSupport
@Override
protected void doShutdown() throws Exception {
+ if (shutdownExecutorService && executorService != null) {
+
getCamelContext().getExecutorServiceManager().shutdownNow(executorService);
+ // an exchange that still waits for a part (its route was stopped
by force) never completes once the thread
+ // pool is shut down, as the queued parts are dropped: fail it, so
its caller does not wait for ever
+ for (MulticastTask task : parallelTasks) {
+ task.doFailed(new RouteStoppingException("The exchange cannot
complete: the thread pool is shut down"));
+ }
+ }
ServiceHelper.stopAndShutdownServices(
processors, errorHandlers != null ? errorHandlers.values() :
null,
aggregationStrategy, processorExchangeFactory);
@@ -1493,9 +1526,6 @@ public class MulticastProcessor extends
BaseProcessorSupport
errorHandlers.clear();
}
- if (shutdownExecutorService && executorService != null) {
-
getCamelContext().getExecutorServiceManager().shutdownNow(executorService);
- }
if (shutdownAggregateExecutorService && aggregateExecutorService !=
null) {
getCamelContext().getExecutorServiceManager().shutdownNow(aggregateExecutorService);
}
diff --git
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/RecipientList.java
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/RecipientList.java
index 476a0f338704..31433bbef718 100644
---
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/RecipientList.java
+++
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/RecipientList.java
@@ -233,12 +233,14 @@ public class RecipientList extends BaseProcessorSupport
implements IdAware, Rout
@Override
protected void doStop() throws Exception {
- ServiceHelper.stopService(producerCache, aggregationStrategy,
recipientListProcessor);
+ ServiceHelper.stopService(recipientListProcessor, producerCache,
aggregationStrategy);
}
@Override
protected void doShutdown() throws Exception {
- ServiceHelper.stopAndShutdownServices(producerCache,
aggregationStrategy, recipientListProcessor);
+ // the processor first, as Multicast and Split do: its shutdown fails
the exchanges that still wait for a part,
+ // and they go through onCompletion of the aggregation strategy and
give their producers back to the cache
+ ServiceHelper.stopAndShutdownServices(recipientListProcessor,
producerCache, aggregationStrategy);
if (aggregateExecutorService != null) {
camelContext.getExecutorServiceManager().shutdownNow(aggregateExecutorService);
diff --git
a/core/camel-core/src/test/java/org/apache/camel/processor/ParallelProcessingRemoveRouteTest.java
b/core/camel-core/src/test/java/org/apache/camel/processor/ParallelProcessingRemoveRouteTest.java
new file mode 100644
index 000000000000..40f5bb21a13a
--- /dev/null
+++
b/core/camel-core/src/test/java/org/apache/camel/processor/ParallelProcessingRemoveRouteTest.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.camel.AggregationStrategy;
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.Exchange;
+import org.apache.camel.RouteStoppingException;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.builder.ThreadPoolProfileBuilder;
+import org.apache.camel.support.service.ServiceSupport;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * A parallel Multicast, Split or Recipient List with its own thread pool,
whose route is stopped after the shutdown
+ * timeout and then removed while a part is still queued in that pool, must
complete (fail) the exchange, not leave it
+ * waiting for the part that the shut down pool never runs.
+ */
+class ParallelProcessingRemoveRouteTest extends ContextTestSupport {
+
+ private final CountDownLatch latch = new CountDownLatch(1);
+ private final CountDownLatch blocked = new CountDownLatch(1);
+
+ @Override
+ public boolean isUseRouteBuilder() {
+ return false;
+ }
+
+ @Override
+ @BeforeEach
+ public void setUp() throws Exception {
+ super.setUp();
+ // one thread in the thread pool of the EIP, so the second part waits
in its queue
+ context.getExecutorServiceManager()
+ .registerThreadPoolProfile(new
ThreadPoolProfileBuilder("one").poolSize(1).maxPoolSize(1).build());
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("direct:block").routeId("block").process(e -> {
+ blocked.countDown();
+ latch.await(20, TimeUnit.SECONDS);
+ });
+ }
+ });
+ }
+
+ @Test
+ void testMulticastRemoveRoute() throws Exception {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("direct:start").routeId("parallel")
+
.multicast().parallelProcessing().executorService("one")
+ .to("direct:block", "mock:b");
+ }
+ });
+ context.start();
+ assertCompletedOnRemove();
+ }
+
+ @Test
+ void testSplitRemoveRoute() throws Exception {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("direct:start").routeId("parallel")
+
.split(body()).parallelProcessing().executorService("one")
+ .toD("${body}");
+ }
+ });
+ context.start();
+ assertCompletedOnRemove();
+ }
+
+ @Test
+ void testRecipientListRemoveRoute() throws Exception {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("direct:start").routeId("parallel")
+
.recipientList(body()).parallelProcessing().executorService("one");
+ }
+ });
+ context.start();
+ assertCompletedOnRemove();
+ }
+
+ @Test
+ void testRecipientListRemoveRouteCompletesBeforeTheStrategyIsShutDown()
throws Exception {
+ RecordingStrategy strategy = new RecordingStrategy();
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ from("direct:start").routeId("parallel")
+
.recipientList(body()).parallelProcessing().executorService("one").aggregationStrategy(strategy);
+ }
+ });
+ context.start();
+ assertCompletedOnRemove();
+
+ assertTrue(strategy.completed, "the failed exchange did not go through
onCompletion of the strategy");
+ assertFalse(strategy.shutDownAtCompletion, "onCompletion ran on a
strategy that was shut down already");
+ }
+
+ @Test
+ void testMulticastReachedAfterTheRouteIsRemoved() throws Exception {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+ // the exchange waits before the Multicast, which then gets it
after its thread pool is shut down. No
+ // error handler: the one of the route would already fail the
exchange, as its route is stopping
+ errorHandler(noErrorHandler());
+ from("direct:start").routeId("parallel")
+ .to("direct:block")
+
.multicast().parallelProcessing().executorService("one")
+ .to("mock:a", "mock:b");
+ }
+ });
+ context.start();
+
+ CompletableFuture<Exchange> future =
template.asyncSend("direct:start", e -> e.getMessage().setBody("A"));
+ assertTrue(blocked.await(5, TimeUnit.SECONDS));
+ try {
+ context.getRouteController().stopRoute("parallel", 100,
TimeUnit.MILLISECONDS);
+ context.removeRoute("parallel");
+ } finally {
+ latch.countDown();
+ }
+
+ // the thread pool drops the task without an exception (CallerRuns
policy), so the Multicast fails the exchange
+ Exchange out = future.get(10, TimeUnit.SECONDS);
+ assertInstanceOf(RouteStoppingException.class, out.getException());
+ await().atMost(5, TimeUnit.SECONDS).until(() ->
context.getInflightRepository().size() == 0);
+ }
+
+ private void assertCompletedOnRemove() throws Exception {
+ CompletableFuture<Exchange> future = template.asyncSend("direct:start",
+ e -> e.getMessage().setBody(List.of("direct:block",
"mock:b")));
+ assertTrue(blocked.await(5, TimeUnit.SECONDS));
+
+ try {
+ // the shutdown timeout runs out, the route is stopped by force,
then removed
+ context.getRouteController().stopRoute("parallel", 100,
TimeUnit.MILLISECONDS);
+ context.removeRoute("parallel");
+
+ Exchange out = future.get(10, TimeUnit.SECONDS);
+ // the shutdown of the EIP fails the exchange. The first part,
which the shutdown of the thread pool
+ // interrupts, fails as well, but it cannot complete the exchange,
as the second part never runs
+ assertInstanceOf(RouteStoppingException.class, out.getException());
+ } finally {
+ latch.countDown();
+ }
+ await().atMost(5, TimeUnit.SECONDS).until(() ->
context.getInflightRepository().size() == 0);
+ }
+
+ /**
+ * Records whether it was shut down when the exchange completed.
+ */
+ private static final class RecordingStrategy extends ServiceSupport
implements AggregationStrategy {
+ private volatile boolean completed;
+ private volatile boolean shutDownAtCompletion;
+
+ @Override
+ public Exchange aggregate(Exchange oldExchange, Exchange newExchange) {
+ return newExchange != null ? newExchange : oldExchange;
+ }
+
+ @Override
+ public void onCompletion(Exchange exchange, Exchange inputExchange) {
+ shutDownAtCompletion = isShutdown();
+ completed = true;
+ }
+ }
+}