This is an automated email from the ASF dual-hosted git repository.

davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git


The following commit(s) were added to refs/heads/main by this push:
     new bcac82ce5374 CAMEL-27088: camel-core - Delay, Throttle, Threads EIP: 
fail the exchanges left in the thread pool when the route is removed (#30304)
bcac82ce5374 is described below

commit bcac82ce5374b5f458eb3de6f5347cd9053488c2
Author: allthingssecurity <[email protected]>
AuthorDate: Wed Nov 18 18:19:16 2026 +0530

    CAMEL-27088: camel-core - Delay, Throttle, Threads EIP: fail the exchanges 
left in the thread pool when the route is removed (#30304)
    
    When a route is stopped after the shutdown timeout and then removed (or 
reloaded), the Delay EIP, the
    Throttle EIP with asyncDelayed and the Threads EIP shut down their own 
thread pool with shutdownNow.
    The exchanges whose task was still queued in that pool were dropped: their 
callback was never called,
    so the caller (such as a ProducerTemplate request) waited for ever, and the 
exchange stayed in the
    in-flight registry.
    
    The Delay and Throttle EIPs now remember their queued tasks; the task and 
the shutdown each claim a
    task by removing it, and after shutting down its own thread pool the 
shutdown fails the exchanges it
    claimed with a RouteStoppingException. The Threads EIP rejects the tasks 
that shutdownNow returns.
    A thread pool of the user is not shut down by the EIP, so its tasks still 
run, as before. The task of
    the Throttle EIP stays a Callable, so the thread pool is called with the 
same submit and schedule
    methods as before.
    
    When doShutdown claimed a call between its registration and a rejected 
schedule or submit, the rejected path
    completed the exchange a second time (and the Delay decremented its counter 
twice): it now leaves the exchange alone
    when it cannot claim the call. When the thread pool of the EIP was shut 
down while the exchange was handed to it, the
    CallerRuns policy dropped the task without an exception: the exchange is 
now failed in the calling thread if it can
    still be claimed. A comment in the Threads EIP notes that wrapped tasks of 
instrumented thread pools are not failed.
    
    Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
 .../apache/camel/processor/AbstractThrottler.java  |  90 +++++++++
 .../processor/ConcurrentRequestsThrottler.java     |  13 +-
 .../camel/processor/DelayProcessorSupport.java     |  47 +++++
 .../apache/camel/processor/ThreadsProcessor.java   |  14 +-
 .../camel/processor/TotalRequestsThrottler.java    |  13 +-
 .../processor/AsyncDelayedRemoveRouteTest.java     | 207 +++++++++++++++++++++
 .../processor/AsyncDelayedShutdownRaceTest.java    | 204 ++++++++++++++++++++
 7 files changed, 583 insertions(+), 5 deletions(-)

diff --git 
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/AbstractThrottler.java
 
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/AbstractThrottler.java
index 06a1b4bf2007..43c234eb5d24 100644
--- 
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/AbstractThrottler.java
+++ 
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/AbstractThrottler.java
@@ -16,6 +16,9 @@
  */
 package org.apache.camel.processor;
 
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -53,6 +56,10 @@ public abstract class AbstractThrottler extends 
BaseProcessorSupport
     protected boolean asyncDelayed;
     protected boolean callerRunsWhenRejected = true;
     protected volatile Expression maxRequestsExpression;
+    // the asynchronous tasks (asyncDelayed) that are queued in the thread 
pool and not yet started: a task and
+    // doShutdown each claim a task by removing it, so an exchange is 
processed, or failed when the thread pool is shut
+    // down, but never both
+    private final Set<AsyncCall> asyncCalls = ConcurrentHashMap.newKeySet();
 
     AbstractThrottler(final ScheduledExecutorService asyncExecutor, final 
boolean shutdownAsyncExecutor,
                       final CamelContext camelContext, final boolean 
rejectExecution, Expression correlation,
@@ -82,6 +89,89 @@ public abstract class AbstractThrottler extends 
BaseProcessorSupport
         }
     }
 
+    /**
+     * A task that processes the exchange again on the thread pool, unless 
doShutdown failed the exchange before.
+     */
+    final class AsyncCall implements Callable<Boolean> {
+        private final Exchange exchange;
+        private final AsyncCallback callback;
+
+        private AsyncCall(Exchange exchange, AsyncCallback callback) {
+            this.exchange = exchange;
+            this.callback = callback;
+        }
+
+        // a Callable, as the task was before, so the thread pool is called 
with the same submit and schedule methods
+        @Override
+        public Boolean call() {
+            if (asyncCalls.remove(this)) {
+                return process(exchange, callback);
+            }
+            return false;
+        }
+
+        /**
+         * The thread pool did not take the task (rejected), so it is no 
longer queued.
+         *
+         * @return true if the caller still owns the exchange, false if 
doShutdown has already failed the exchange and
+         *         called the callback (the caller must then leave the 
exchange alone)
+         */
+        boolean rejected() {
+            return asyncCalls.remove(this);
+        }
+
+        private void fail() {
+            // the exchange is no longer queued (a later throttler must 
process it synchronously)
+            exchange.removeProperty(PROPERTY_EXCHANGE_STATE);
+            exchange.setException(
+                    new RouteStoppingException("Throttled exchange is not 
processed as the throttler is shut down"));
+        }
+    }
+
+    /**
+     * Creates the task to process the exchange again on the thread pool, and 
remembers it until it starts.
+     */
+    AsyncCall newAsyncCall(Exchange exchange, AsyncCallback callback) {
+        AsyncCall call = new AsyncCall(exchange, callback);
+        asyncCalls.add(call);
+        return call;
+    }
+
+    /**
+     * Fails the exchanges whose asynchronous task has not started yet. They 
would never be processed once the thread
+     * pool is shut down (such as when the route is removed after the shutdown 
timeout ran out), and their callers would
+     * wait for ever. Only called once doShutdown shut down the thread pool of 
this EIP: a thread pool of the user still
+     * runs the tasks.
+     */
+    void failAsyncCalls() {
+        for (AsyncCall call : asyncCalls) {
+            if (asyncCalls.remove(call)) {
+                LOG.debug("Throttled exchange is not processed as the 
throttler is shut down, exchangeId: {}",
+                        call.exchange.getExchangeId());
+                call.fail();
+                call.callback.done(false);
+            }
+        }
+    }
+
+    /**
+     * Fails the exchange in the calling thread when the thread pool of this 
EIP was shut down while the task was handed
+     * to it. With the CallerRuns rejected policy (the default) a thread pool 
that is shut down drops the task without
+     * an exception, and doShutdown may have failed the queued tasks before 
this one was added.
+     *
+     * @return true if the exchange was failed here and the callback was 
called synchronously
+     */
+    boolean failIfShutdown(AsyncCall call) {
+        if (shutdownAsyncExecutor && asyncExecutor.isShutdown() && 
asyncCalls.remove(call)) {
+            LOG.debug("Throttled exchange is not processed as the throttler is 
shut down, exchangeId: {}",
+                    call.exchange.getExchangeId());
+            call.fail();
+            call.callback.done(true);
+            return true;
+        }
+        return false;
+    }
+
     protected static boolean handleInterrupt(
             Exchange exchange, AsyncCallback callback, InterruptedException e, 
boolean doneSync) {
         // determine if we can still run, or the camel context is forcing a 
shutdown
diff --git 
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/ConcurrentRequestsThrottler.java
 
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/ConcurrentRequestsThrottler.java
index f1d55bfc0ef2..e5eff47cf5e2 100644
--- 
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/ConcurrentRequestsThrottler.java
+++ 
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/ConcurrentRequestsThrottler.java
@@ -183,14 +183,20 @@ public class ConcurrentRequestsThrottler extends 
AbstractThrottler {
      */
     protected boolean processAsynchronously(
             final Exchange exchange, final AsyncCallback callback, 
ThrottlingState throttlingState) {
+        AsyncCall call = newAsyncCall(exchange, callback);
         try {
             if (LOG.isTraceEnabled()) {
                 exchange.setProperty(PROPERTY_EXCHANGE_QUEUED_TIME, 
TimeSource.nanoTime());
             }
             exchange.setProperty(PROPERTY_EXCHANGE_STATE, State.ASYNC);
-            asyncExecutor.submit(() -> process(exchange, callback));
-            return false;
+            asyncExecutor.submit(call);
+            // a thread pool that is shut down drops the task without an 
exception (CallerRuns)
+            return failIfShutdown(call);
         } catch (final RejectedExecutionException e) {
+            if (!call.rejected()) {
+                // doShutdown has already failed the exchange and called the 
callback
+                return false;
+            }
             if (isCallerRunsWhenRejected()) {
                 if (LOG.isDebugEnabled()) {
                     LOG.debug("AsyncExecutor is full, rejected exchange will 
run in the current thread, exchangeId: {}",
@@ -214,6 +220,9 @@ public class ConcurrentRequestsThrottler extends 
AbstractThrottler {
     protected void doShutdown() throws Exception {
         if (shutdownAsyncExecutor && asyncExecutor != null) {
             
camelContext.getExecutorServiceManager().shutdownNow(asyncExecutor);
+            // the tasks still queued in the thread pool that was shut down 
never run (a thread pool of the user
+            // is not shut down here, and still runs them)
+            failAsyncCalls();
         }
         states.clear();
         super.doShutdown();
diff --git 
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/DelayProcessorSupport.java
 
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/DelayProcessorSupport.java
index d5b2c9731658..c6f0838fc38f 100644
--- 
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/DelayProcessorSupport.java
+++ 
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/DelayProcessorSupport.java
@@ -16,6 +16,8 @@
  */
 package org.apache.camel.processor;
 
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -46,6 +48,9 @@ public abstract class DelayProcessorSupport extends 
BaseDelegateProcessorSupport
     private boolean asyncDelayed = true;
     private boolean callerRunsWhenRejected = true;
     private final AtomicInteger delayedCount = new AtomicInteger();
+    // the delayed tasks that are scheduled and not yet started: a task and 
doShutdown each claim a task by removing
+    // it, so an exchange is routed on, or failed when the thread pool is shut 
down, but never both
+    private final Set<ProcessCall> scheduledCalls = 
ConcurrentHashMap.newKeySet();
 
     private final class ProcessCall implements Runnable {
         private final Exchange exchange;
@@ -58,6 +63,10 @@ public abstract class DelayProcessorSupport extends 
BaseDelegateProcessorSupport
 
         @Override
         public void run() {
+            if (!scheduledCalls.remove(this)) {
+                // the exchange was already failed by doShutdown
+                return;
+            }
             // we are running now so decrement the counter
             delayedCount.decrementAndGet();
 
@@ -120,15 +129,29 @@ public abstract class DelayProcessorSupport extends 
BaseDelegateProcessorSupport
             // and increment the counter (we decrement the counter when we run 
the ProcessCall)
             delayedCount.incrementAndGet();
             ProcessCall call = new ProcessCall(exchange, callback);
+            scheduledCalls.add(call);
             try {
                 if (LOG.isTraceEnabled()) {
                     LOG.trace("Scheduling delayed task to run in {} millis for 
exchangeId: {}",
                             delay, exchange.getExchangeId());
                 }
                 executorService.schedule(call, delay, TimeUnit.MILLISECONDS);
+                if (shutdownExecutorService && executorService.isShutdown() && 
scheduledCalls.remove(call)) {
+                    // the thread pool was shut down while the task was handed 
to it: with the CallerRuns rejected
+                    // policy (the default) it drops the task without an 
exception, and doShutdown may have failed
+                    // the scheduled tasks before this one was added
+                    delayedCount.decrementAndGet();
+                    failNotRun(exchange);
+                    callback.done(true);
+                    return true;
+                }
                 // tell Camel routing engine we continue routing asynchronous
                 return false;
             } catch (RejectedExecutionException e) {
+                if (!scheduledCalls.remove(call)) {
+                    // doShutdown has already failed the exchange and called 
the callback
+                    return false;
+                }
                 // we were not allowed to run the ProcessCall, so need to 
decrement the counter here
                 delayedCount.decrementAndGet();
                 if (isCallerRunsWhenRejected()) {
@@ -273,10 +296,34 @@ public abstract class DelayProcessorSupport extends 
BaseDelegateProcessorSupport
         super.doStart();
     }
 
+    /**
+     * Fails the exchanges whose delayed task has not started yet. They would 
never be routed on once the thread pool is
+     * shut down (such as when the route is removed after the shutdown timeout 
ran out), and their callers would wait
+     * for ever. Only called once doShutdown shut down the thread pool of this 
EIP: a thread pool of the user still runs
+     * the tasks.
+     */
+    private void failScheduledCalls() {
+        for (ProcessCall call : scheduledCalls) {
+            if (scheduledCalls.remove(call)) {
+                delayedCount.decrementAndGet();
+                failNotRun(call.exchange);
+                call.callback.done(false);
+            }
+        }
+    }
+
+    private static void failNotRun(Exchange exchange) {
+        LOG.debug("Delayed task is not run as the processor is shut down, for 
exchangeId: {}", exchange.getExchangeId());
+        exchange.setException(new RouteStoppingException("The delayed exchange 
is not run: the Delay is shut down"));
+    }
+
     @Override
     protected void doShutdown() throws Exception {
         if (shutdownExecutorService && executorService != null) {
             
camelContext.getExecutorServiceManager().shutdownNow(executorService);
+            // the tasks still queued in the thread pool that was shut down 
never run (a thread pool of the user
+            // is not shut down here, and still runs them)
+            failScheduledCalls();
         }
         super.doShutdown();
     }
diff --git 
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/ThreadsProcessor.java
 
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/ThreadsProcessor.java
index c2cbe8ae1df2..c230e7cba539 100644
--- 
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/ThreadsProcessor.java
+++ 
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/ThreadsProcessor.java
@@ -16,6 +16,7 @@
  */
 package org.apache.camel.processor;
 
+import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -208,7 +209,18 @@ public class ThreadsProcessor extends BaseProcessorSupport 
implements IdAware, R
     @Override
     protected void doShutdown() throws Exception {
         if (shutdownExecutorService) {
-            
camelContext.getExecutorServiceManager().shutdownNow(executorService);
+            List<Runnable> dropped = 
camelContext.getExecutorServiceManager().shutdownNow(executorService);
+            if (dropped != null) {
+                // the exchanges still queued in the thread pool are never 
processed: reject them so their callers
+                // do not wait for ever. A thread pool factory that wraps the 
tasks (such as the instrumented thread
+                // pools of camel-opentelemetry or camel-micrometer) hands 
back its own wrappers here, which are not
+                // Rejectable, so those exchanges are not failed here
+                for (Runnable task : dropped) {
+                    if (task instanceof Rejectable rejectable) {
+                        rejectable.reject();
+                    }
+                }
+            }
         }
         super.doShutdown();
     }
diff --git 
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/TotalRequestsThrottler.java
 
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/TotalRequestsThrottler.java
index 8a28c1809483..688cbfe901ec 100644
--- 
a/core/camel-core-processor/src/main/java/org/apache/camel/processor/TotalRequestsThrottler.java
+++ 
b/core/camel-core-processor/src/main/java/org/apache/camel/processor/TotalRequestsThrottler.java
@@ -191,6 +191,7 @@ public class TotalRequestsThrottler extends 
AbstractThrottler {
      */
     protected boolean processAsynchronously(
             final Exchange exchange, final AsyncCallback callback, 
ThrottlingState throttlingState) {
+        AsyncCall call = newAsyncCall(exchange, callback);
         try {
             if (LOG.isTraceEnabled()) {
                 exchange.setProperty(PROPERTY_EXCHANGE_QUEUED_TIMESTAMP, 
TimeSource.currentTimeMillis());
@@ -201,9 +202,14 @@ public class TotalRequestsThrottler extends 
AbstractThrottler {
             // exchanges and not yet returned, so try again after one period
             long delay = next != null
                     ? next.getDelay(TimeUnit.NANOSECONDS) : 
TimeUnit.MILLISECONDS.toNanos(getTimePeriodMillis());
-            asyncExecutor.schedule(() -> process(exchange, callback), delay, 
TimeUnit.NANOSECONDS);
-            return false;
+            asyncExecutor.schedule(call, delay, TimeUnit.NANOSECONDS);
+            // a thread pool that is shut down drops the task without an 
exception (CallerRuns)
+            return failIfShutdown(call);
         } catch (final RejectedExecutionException e) {
+            if (!call.rejected()) {
+                // doShutdown has already failed the exchange and called the 
callback
+                return false;
+            }
             if (isCallerRunsWhenRejected()) {
                 if (LOG.isDebugEnabled()) {
                     LOG.debug("AsyncExecutor is full, rejected exchange will 
run in the current thread, exchangeId: {}",
@@ -227,6 +233,9 @@ public class TotalRequestsThrottler extends 
AbstractThrottler {
     protected void doShutdown() throws Exception {
         if (shutdownAsyncExecutor && asyncExecutor != null) {
             
camelContext.getExecutorServiceManager().shutdownNow(asyncExecutor);
+            // the tasks still queued in the thread pool that was shut down 
never run (a thread pool of the user
+            // is not shut down here, and still runs them)
+            failAsyncCalls();
         }
         states.clear();
         super.doShutdown();
diff --git 
a/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedRemoveRouteTest.java
 
b/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedRemoveRouteTest.java
new file mode 100644
index 000000000000..c44f7636d9d6
--- /dev/null
+++ 
b/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedRemoveRouteTest.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.Exchange;
+import org.apache.camel.RouteStoppingException;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.builder.ThreadPoolProfileBuilder;
+import org.apache.camel.spi.ThreadPoolProfile;
+import org.junit.jupiter.api.Test;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * An exchange that waits in the thread pool of a Delay, Throttle 
(asyncDelayed) or Threads EIP when its route is
+ * stopped after the shutdown timeout and then removed must be completed 
(failed), not dropped with the thread pool. A
+ * thread pool of the user is not shut down with the route, so its tasks still 
run and complete the exchange.
+ */
+class AsyncDelayedRemoveRouteTest extends ContextTestSupport {
+
+    private final CountDownLatch latch = new CountDownLatch(1);
+
+    @Override
+    public boolean isUseRouteBuilder() {
+        return false;
+    }
+
+    @Test
+    void testDelayRemoveRoute() throws Exception {
+        context.addRoutes(new RouteBuilder() {
+            @Override
+            public void configure() {
+                
from("direct:delay").routeId("delay").delay(60000).asyncDelayed().to("mock:result");
+            }
+        });
+        context.start();
+        assertExchangeCompletedOnRemove("direct:delay", "delay");
+    }
+
+    @Test
+    void testThrottleRemoveRoute() throws Exception {
+        context.addRoutes(new RouteBuilder() {
+            @Override
+            public void configure() {
+                from("direct:throttle").routeId("throttle")
+                        
.throttle(1).totalRequestsMode().timePeriodMillis(60000).asyncDelayed()
+                        .to("mock:result");
+            }
+        });
+        context.start();
+        // the first exchange takes the only permit of the period
+        template.sendBody("direct:throttle", "first");
+        assertExchangeCompletedOnRemove("direct:throttle", "throttle");
+    }
+
+    @Test
+    void testConcurrentThrottleRemoveRoute() throws Exception {
+        // one thread in the thread pool of the throttler
+        ThreadPoolProfile one = new 
ThreadPoolProfileBuilder("one").poolSize(1).maxPoolSize(1).build();
+        context.getExecutorServiceManager().registerThreadPoolProfile(one);
+        context.addRoutes(new RouteBuilder() {
+            @Override
+            public void configure() {
+                from("direct:concurrent").routeId("concurrent")
+                        
.throttle(1).concurrentRequestsMode().asyncDelayed().executorService("one")
+                        .process(e -> {
+                            if ("first".equals(e.getMessage().getBody())) {
+                                latch.await(20, TimeUnit.SECONDS);
+                            }
+                        });
+            }
+        });
+        context.start();
+        // the first exchange keeps the only permit, the second one waits for 
it on the only thread of the pool
+        CompletableFuture<Exchange> first = 
template.asyncSend("direct:concurrent", e -> e.getMessage().setBody("first"));
+        await().atMost(5, TimeUnit.SECONDS).until(() -> 
context.getInflightRepository().size("concurrent") == 1);
+        CompletableFuture<Exchange> second = 
template.asyncSend("direct:concurrent", e -> e.getMessage().setBody("second"));
+        await().atMost(5, TimeUnit.SECONDS).until(() -> 
context.getInflightRepository().size("concurrent") == 2);
+        try {
+            // the third one is queued in the thread pool
+            assertExchangeCompletedOnRemove("direct:concurrent", "concurrent", 
3);
+            assertNotNull(second.get(10, TimeUnit.SECONDS).getException());
+        } finally {
+            latch.countDown();
+        }
+        // the first one completes too, cut off by the forced stop
+        assertInstanceOf(RouteStoppingException.class, first.get(10, 
TimeUnit.SECONDS).getException());
+        assertEquals(0, context.getInflightRepository().size());
+    }
+
+    @Test
+    void testThreadsRemoveRoute() throws Exception {
+        context.addRoutes(new RouteBuilder() {
+            @Override
+            public void configure() {
+                from("direct:threads").routeId("threads")
+                        .threads(1, 1).maxQueueSize(10)
+                        .process(e -> {
+                            if ("first".equals(e.getMessage().getBody())) {
+                                latch.await(20, TimeUnit.SECONDS);
+                            }
+                        });
+            }
+        });
+        context.start();
+        // the first exchange keeps the only thread of the pool busy
+        CompletableFuture<Exchange> first = 
template.asyncSend("direct:threads", e -> e.getMessage().setBody("first"));
+        await().atMost(5, TimeUnit.SECONDS).until(() -> 
context.getInflightRepository().size("threads") == 1);
+        try {
+            // the second one is queued in the thread pool
+            assertExchangeCompletedOnRemove("direct:threads", "threads", 2);
+        } finally {
+            latch.countDown();
+        }
+        // the first one was interrupted by the shutdown of the thread pool
+        assertNotNull(first.get(10, TimeUnit.SECONDS).getException());
+        assertEquals(0, context.getInflightRepository().size());
+    }
+
+    @Test
+    void testDelayWithThreadPoolOfTheUserRemoveRoute() throws Exception {
+        ScheduledExecutorService pool = 
Executors.newSingleThreadScheduledExecutor();
+        try {
+            context.getRegistry().bind("userPool", pool);
+            context.addRoutes(new RouteBuilder() {
+                @Override
+                public void configure() {
+                    
from("direct:user").routeId("user").delay(10).asyncDelayed().executorService("userPool")
+                            .to("mock:result");
+                }
+            });
+            context.start();
+            // the only thread of the pool is busy, so the delayed task waits 
in its queue
+            CountDownLatch busy = new CountDownLatch(1);
+            pool.execute(() -> {
+                busy.countDown();
+                try {
+                    latch.await(20, TimeUnit.SECONDS);
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                }
+            });
+            assertTrue(busy.await(5, TimeUnit.SECONDS));
+            CompletableFuture<Exchange> future = 
template.asyncSend("direct:user", e -> e.getMessage().setBody("waiting"));
+            await().atMost(5, TimeUnit.SECONDS).until(() -> 
context.getInflightRepository().size("user") == 1);
+
+            context.getRouteController().stopRoute("user", 100, 
TimeUnit.MILLISECONDS);
+            context.removeRoute("user");
+
+            // the route does not shut down the thread pool of the user, so 
the task is still queued there and is not
+            // failed by the removal
+            await().during(300, TimeUnit.MILLISECONDS).atMost(2, 
TimeUnit.SECONDS).until(() -> !future.isDone());
+            assertFalse(pool.isShutdown());
+
+            // the task runs once the thread is free, and fails the exchange 
as its route was removed
+            latch.countDown();
+            assertInstanceOf(RouteStoppingException.class, future.get(10, 
TimeUnit.SECONDS).getException());
+            assertEquals(0, context.getInflightRepository().size());
+        } finally {
+            latch.countDown();
+            pool.shutdownNow();
+        }
+    }
+
+    private void assertExchangeCompletedOnRemove(String uri, String routeId) 
throws Exception {
+        assertExchangeCompletedOnRemove(uri, routeId, 1);
+        assertEquals(0, context.getInflightRepository().size());
+    }
+
+    private void assertExchangeCompletedOnRemove(String uri, String routeId, 
int inflight) throws Exception {
+        CompletableFuture<Exchange> future = template.asyncSend(uri, e -> 
e.getMessage().setBody("waiting"));
+        await().atMost(5, TimeUnit.SECONDS).until(() -> 
context.getInflightRepository().size(routeId) == inflight);
+
+        // the shutdown timeout runs out, the route is stopped by force, then 
removed
+        context.getRouteController().stopRoute(routeId, 100, 
TimeUnit.MILLISECONDS);
+        context.removeRoute(routeId);
+
+        Exchange out = future.get(10, TimeUnit.SECONDS);
+        assertInstanceOf(RouteStoppingException.class, out.getException());
+    }
+}
diff --git 
a/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedShutdownRaceTest.java
 
b/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedShutdownRaceTest.java
new file mode 100644
index 000000000000..d476b7531c51
--- /dev/null
+++ 
b/core/camel-core/src/test/java/org/apache/camel/processor/AsyncDelayedShutdownRaceTest.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.Exchange;
+import org.apache.camel.RouteStoppingException;
+import org.apache.camel.Service;
+import org.apache.camel.support.DefaultExchange;
+import org.apache.camel.support.builder.ExpressionBuilder;
+import org.apache.camel.support.service.ServiceHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The thread pool of a Delay or Throttle (asyncDelayed) EIP is shut down by 
the EIP while an exchange is being handed
+ * to it. The exchange must be completed exactly once: either by doShutdown, 
or by the thread that hands it over, never
+ * by both, and never by none.
+ */
+class AsyncDelayedShutdownRaceTest extends ContextTestSupport {
+
+    private final AtomicInteger done = new AtomicInteger();
+    private Exchange exchange;
+    private HookPool pool;
+
+    @BeforeEach
+    void createExchange() {
+        exchange = new DefaultExchange(context);
+    }
+
+    @AfterEach
+    void shutdownPool() {
+        if (pool != null) {
+            pool.shutdownNow();
+        }
+    }
+
+    @Override
+    public boolean isUseRouteBuilder() {
+        return false;
+    }
+
+    @Test
+    void testDelayRejectedAfterShutdownFailedTheExchange() throws Exception {
+        // the EIP is shut down after the task was registered and before the 
pool takes it, so doShutdown fails the
+        // exchange, then the pool that is shut down rejects the task
+        pool = new HookPool(new ThreadPoolExecutor.AbortPolicy());
+        Delayer delayer = new Delayer(context, e -> {
+        }, ExpressionBuilder.constantExpression(60000L), pool, true);
+        ServiceHelper.startService(delayer);
+        pool.beforeSchedule = () -> 
ServiceHelper.stopAndShutdownService(delayer);
+
+        boolean sync = delayer.process(exchange, doneSync -> 
done.incrementAndGet());
+
+        assertFalse(sync);
+        assertCompletedOnce();
+        assertEquals(0, delayer.getDelayedCount());
+    }
+
+    @Test
+    void testDelayDroppedByShutDownPool() throws Exception {
+        // doShutdown has failed the scheduled tasks before this one was 
registered, and the pool that is shut down
+        // drops it without an exception (CallerRuns)
+        pool = new HookPool(new ThreadPoolExecutor.CallerRunsPolicy());
+        Delayer delayer = new Delayer(context, e -> {
+        }, ExpressionBuilder.constantExpression(60000L), pool, true);
+        ServiceHelper.startService(delayer);
+        ServiceHelper.stopAndShutdownService(delayer);
+
+        boolean sync = delayer.processDelay(exchange, doneSync -> 
done.incrementAndGet(), 60000L);
+
+        assertTrue(sync);
+        assertCompletedOnce();
+        assertEquals(0, delayer.getDelayedCount());
+    }
+
+    @Test
+    void testConcurrentThrottlerRejectedAfterShutdownFailedTheExchange() 
throws Exception {
+        pool = new HookPool(new ThreadPoolExecutor.AbortPolicy());
+        ConcurrentRequestsThrottler throttler = new 
ConcurrentRequestsThrottler(
+                context, ExpressionBuilder.constantExpression(1), pool, true, 
false, null);
+        throttler.setAsyncDelayed(true);
+        assertRejectedAfterShutdownCompletesOnce(throttler,
+                () -> throttler.processAsynchronously(exchange, doneSync -> 
done.incrementAndGet(), null));
+    }
+
+    @Test
+    void testTotalThrottlerRejectedAfterShutdownFailedTheExchange() throws 
Exception {
+        pool = new HookPool(new ThreadPoolExecutor.AbortPolicy());
+        TotalRequestsThrottler throttler = new TotalRequestsThrottler(
+                context, ExpressionBuilder.constantExpression(1), 60000L, 
pool, true, false, null);
+        throttler.setAsyncDelayed(true);
+        TotalRequestsThrottler.ThrottlingState state = throttler.new 
ThrottlingState("test");
+        assertRejectedAfterShutdownCompletesOnce(throttler,
+                () -> throttler.processAsynchronously(exchange, doneSync -> 
done.incrementAndGet(), state));
+    }
+
+    @Test
+    void testConcurrentThrottlerDroppedByShutDownPool() throws Exception {
+        pool = new HookPool(new ThreadPoolExecutor.CallerRunsPolicy());
+        ConcurrentRequestsThrottler throttler = new 
ConcurrentRequestsThrottler(
+                context, ExpressionBuilder.constantExpression(1), pool, true, 
false, null);
+        throttler.setAsyncDelayed(true);
+        assertDroppedByShutDownPoolCompletesOnce(throttler,
+                () -> throttler.processAsynchronously(exchange, doneSync -> 
done.incrementAndGet(), null));
+    }
+
+    @Test
+    void testTotalThrottlerDroppedByShutDownPool() throws Exception {
+        pool = new HookPool(new ThreadPoolExecutor.CallerRunsPolicy());
+        TotalRequestsThrottler throttler = new TotalRequestsThrottler(
+                context, ExpressionBuilder.constantExpression(1), 60000L, 
pool, true, false, null);
+        throttler.setAsyncDelayed(true);
+        TotalRequestsThrottler.ThrottlingState state = throttler.new 
ThrottlingState("test");
+        assertDroppedByShutDownPoolCompletesOnce(throttler,
+                () -> throttler.processAsynchronously(exchange, doneSync -> 
done.incrementAndGet(), state));
+    }
+
+    private void assertRejectedAfterShutdownCompletesOnce(Service throttler, 
Callable<Boolean> handOver)
+            throws Exception {
+        ServiceHelper.startService(throttler);
+        pool.beforeSchedule = () -> 
ServiceHelper.stopAndShutdownService(throttler);
+
+        boolean sync = handOver.call();
+
+        assertFalse(sync);
+        assertCompletedOnce();
+    }
+
+    private void assertDroppedByShutDownPoolCompletesOnce(Service throttler, 
Callable<Boolean> handOver)
+            throws Exception {
+        ServiceHelper.startService(throttler);
+        ServiceHelper.stopAndShutdownService(throttler);
+
+        boolean sync = handOver.call();
+
+        assertTrue(sync);
+        assertCompletedOnce();
+    }
+
+    private void assertCompletedOnce() {
+        assertEquals(1, done.get(), "the callback must be called exactly 
once");
+        assertInstanceOf(RouteStoppingException.class, 
exchange.getException());
+    }
+
+    /**
+     * A scheduled thread pool that runs a hook just before it takes a task.
+     */
+    private static final class HookPool extends ScheduledThreadPoolExecutor {
+        private volatile Runnable beforeSchedule;
+
+        HookPool(RejectedExecutionHandler handler) {
+            super(1, handler);
+        }
+
+        @Override
+        public ScheduledFuture<?> schedule(Runnable command, long delay, 
TimeUnit unit) {
+            runHook();
+            return super.schedule(command, delay, unit);
+        }
+
+        @Override
+        public <V> ScheduledFuture<V> schedule(Callable<V> callable, long 
delay, TimeUnit unit) {
+            runHook();
+            return super.schedule(callable, delay, unit);
+        }
+
+        private void runHook() {
+            Runnable hook = beforeSchedule;
+            beforeSchedule = null;
+            if (hook != null) {
+                hook.run();
+            }
+        }
+    }
+}

Reply via email to