This is an automated email from the ASF dual-hosted git repository.

davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git


The following commit(s) were added to refs/heads/main by this push:
     new 5df06208bca8 CAMEL-26970: camel-azure-servicebus - an exchange that 
completes during a graceful stop cannot settle its message: deferShutdown 
closes the receiver (#30131)
5df06208bca8 is described below

commit 5df06208bca8b7bca002193f02ad7ab4a25938f2
Author: allthingssecurity <[email protected]>
AuthorDate: Mon Nov 16 18:59:19 2026 +0530

    CAMEL-26970: camel-azure-servicebus - an exchange that completes during a 
graceful stop cannot settle its message: deferShutdown closes the receiver 
(#30131)
    
    Follow-up of CAMEL-23260. Found by the Service Bus DST world in #30130 
(CAMEL-26956; this PR pairs with it): its default set accepts this case, and 
`-Ddst.servicebus.strictClose=true` fails 96 of 1000 seeds on main. Confirmed 
outside the DST harness against the SDK classes and with a plain unit test.
    
    CAMEL-23260 made `ServiceBusConsumer` `ShutdownAware` so that the exchanges 
in flight at a graceful stop can still settle their message. Its 
`deferShutdown` calls `client.stop()`, with the comment "stop accepting new 
messages but keep the connection open so that in-flight exchanges can still 
complete/abandon messages". With azure-messaging-servicebus 7.17 (javap of 
7.17.16) that is not what `stop()` does:
    - `ServiceBusProcessorClient.stop()` calls `ServiceBusProcessor.stop()` 
when the processor runs on the V2 stack;
    - the V2 stack is the default for a non-session processor 
(`V2StackSupport.isNonSessionAsyncReceiveEnabled` is true unless 
`com.azure.messaging.servicebus.nonSession.asyncReceive.v2` opts out);
    - `ServiceBusProcessor.stop()` is `close()`: it disposes the 
`RollingMessagePump`, a `Mono.using` whose cleanup closes its 
`ServiceBusReceiverAsyncClient`;
    - a settlement through a disposed receiver fails: `updateDisposition` 
returns `IllegalStateException` "Cannot perform operation '...' on a disposed 
receiver".
    
    Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
 .../camel-azure/camel-azure-servicebus/pom.xml     |   6 +
 .../azure/servicebus/ServiceBusConsumer.java       |  66 +++++++++--
 .../azure/servicebus/ServiceBusConsumerTest.java   | 121 ++++++++++++++++++++-
 3 files changed, 182 insertions(+), 11 deletions(-)

diff --git a/components/camel-azure/camel-azure-servicebus/pom.xml 
b/components/camel-azure/camel-azure-servicebus/pom.xml
index 4e584cec43cb..9687ce9b3207 100644
--- a/components/camel-azure/camel-azure-servicebus/pom.xml
+++ b/components/camel-azure/camel-azure-servicebus/pom.xml
@@ -75,6 +75,12 @@
             <version>${mockito-version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.awaitility</groupId>
+            <artifactId>awaitility</artifactId>
+            <version>${awaitility-version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.camel</groupId>
             <artifactId>camel-test-infra-core</artifactId>
diff --git 
a/components/camel-azure/camel-azure-servicebus/src/main/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumer.java
 
b/components/camel-azure/camel-azure-servicebus/src/main/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumer.java
index c2c5a96c4949..147f9da3902f 100644
--- 
a/components/camel-azure/camel-azure-servicebus/src/main/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumer.java
+++ 
b/components/camel-azure/camel-azure-servicebus/src/main/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumer.java
@@ -22,6 +22,7 @@ import java.time.OffsetDateTime;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
@@ -70,6 +71,11 @@ public class ServiceBusConsumer extends DefaultConsumer 
implements ShutdownAware
     private ServiceBusReceiverAsyncClient renewalClient;
     private LockRenewer lockRenewer;
     private final AtomicInteger pendingExchanges = new AtomicInteger();
+    // Set by deferShutdown in peek-lock mode: a graceful stop waits for the 
exchanges in flight, which settle their
+    // message through the processor client, so the client stays open and the 
messages it delivers meanwhile are held
+    // (not routed) until the consumer stops
+    private volatile boolean deferredStop;
+    private volatile CountDownLatch stopped = new CountDownLatch(1);
 
     public ServiceBusConsumer(final ServiceBusEndpoint endpoint, final 
Processor processor) {
         super(endpoint, processor);
@@ -85,6 +91,9 @@ public class ServiceBusConsumer extends DefaultConsumer 
implements ShutdownAware
     protected void doStart() throws Exception {
         super.doStart();
 
+        deferredStop = false;
+        stopped = new CountDownLatch(1);
+
         LOG.debug("Creating connection to Azure ServiceBus");
 
         // close any previously created client (e.g. on route restart after 
doStop)
@@ -129,7 +138,16 @@ public class ServiceBusConsumer extends DefaultConsumer 
implements ShutdownAware
     }
 
     private void processMessage(ServiceBusReceivedMessageContext 
messageContext) {
+        // the latch of this run of the consumer, taken before the check, so a 
stop and start meanwhile cannot leave the
+        // message waiting for a later stop
+        CountDownLatch stop = stopped;
         pendingExchanges.incrementAndGet();
+        // counted before the check: a graceful stop that does not see this 
exchange as pending makes it be held here
+        if (deferredStop) {
+            pendingExchanges.decrementAndGet();
+            holdUntilStopped(stop);
+            return;
+        }
         try {
             final ServiceBusReceivedMessage message = 
messageContext.getMessage();
             final Exchange exchange = createServiceBusExchange(message);
@@ -156,6 +174,22 @@ public class ServiceBusConsumer extends DefaultConsumer 
implements ShutdownAware
         }
     }
 
+    /**
+     * Holds a message delivered during a graceful stop, without routing or 
settling it: the processor client delivers
+     * the next message only when processMessage returns, so at most 
maxConcurrentCalls messages are held (per session
+     * with sessions). The SDK renews their lock while they are held, up to 
maxAutoLockRenewDuration: with 0 the lock of
+     * a held message can expire during the graceful wait, and Service Bus 
then delivers it again to a free call, where
+     * it is held again (one more delivery count each time). They are released 
when the stop has closed the client:
+     * Service Bus delivers a held message again once its lock expires.
+     */
+    private static void holdUntilStopped(CountDownLatch stop) {
+        try {
+            stop.await();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+    }
+
     private void processError(ServiceBusErrorContext errorContext) {
         final Exchange exchange = createServiceBusExchange(errorContext);
 
@@ -176,14 +210,19 @@ public class ServiceBusConsumer extends DefaultConsumer 
implements ShutdownAware
             renewalClient.close();
             renewalClient = null;
         }
-        if (client != null) {
-            if (clientCloseable) {
-                client.close();
-                client = null;
-            } else {
-                // user-provided client: stop accepting messages but don't 
close
-                client.stop();
+        try {
+            if (client != null) {
+                if (clientCloseable) {
+                    client.close();
+                    client = null;
+                } else {
+                    // user-provided client: stop accepting messages but don't 
close
+                    client.stop();
+                }
             }
+        } finally {
+            // the client is closed or stopped: release the messages held 
during a graceful stop
+            stopped.countDown();
         }
 
         // shutdown camel consumer
@@ -205,15 +244,22 @@ public class ServiceBusConsumer extends DefaultConsumer 
implements ShutdownAware
             client.close();
             client = null;
         }
+        stopped.countDown();
 
         super.doShutdown();
     }
 
     @Override
     public boolean deferShutdown(ShutdownRunningTask shutdownRunningTask) {
-        // stop accepting new messages but keep the connection open
-        // so that in-flight exchanges can still complete/abandon messages
-        if (client != null) {
+        if (clientCloseable && getConfiguration().getServiceBusReceiveMode() 
== ServiceBusReceiveMode.PEEK_LOCK) {
+            // stop routing new messages but keep the client open, so that 
in-flight exchanges can still
+            // complete/abandon their message: the stop() of the processor 
client closes its receiver (the default
+            // V2 stack of the SDK), and a message cannot be settled through a 
closed receiver. If a stop with
+            // abortAfterTimeout gives up, the consumer stays in this state 
until the route is stopped and started
+            deferredStop = true;
+        } else if (client != null) {
+            // receive and delete mode: nothing to settle; a processorClient 
given by the user: its messages do not go
+            // through processMessage. Stop receiving
             client.stop();
         }
         return true;
diff --git 
a/components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumerTest.java
 
b/components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumerTest.java
index 863134b0f81e..005ecc5ffbc7 100644
--- 
a/components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumerTest.java
+++ 
b/components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumerTest.java
@@ -20,6 +20,8 @@ import java.time.Duration;
 import java.time.OffsetDateTime;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
 import com.azure.core.amqp.models.AmqpAnnotatedMessage;
@@ -40,12 +42,14 @@ import org.apache.camel.spi.UuidGenerator;
 import org.apache.camel.support.DefaultExchange;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.EnumSource;
 import org.mockito.ArgumentCaptor;
 import reactor.core.publisher.Mono;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.*;
@@ -794,17 +798,132 @@ public class ServiceBusConsumerTest {
     }
 
     @Test
-    void deferShutdownStopsClientAndReturnsTrue() throws Exception {
+    void deferShutdownStopsUserProvidedClient() throws Exception {
+        // its messages go to the callbacks the user built it with, not 
through the consumer
+        when(configuration.getProcessorClient()).thenReturn(client);
+        
when(configuration.getServiceBusReceiveMode()).thenReturn(ServiceBusReceiveMode.PEEK_LOCK);
+        try (ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint, 
processor)) {
+            consumer.doStart();
+
+            
assertThat(consumer.deferShutdown(ShutdownRunningTask.CompleteCurrentTaskOnly)).isTrue();
+            verify(client).stop();
+        }
+    }
+
+    @Test
+    void deferShutdownStopsClientInReceiveAndDeleteMode() throws Exception {
+        
when(configuration.getServiceBusReceiveMode()).thenReturn(ServiceBusReceiveMode.RECEIVE_AND_DELETE);
         try (ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint, 
processor)) {
             consumer.doStart();
 
             boolean deferred = 
consumer.deferShutdown(ShutdownRunningTask.CompleteAllTasks);
 
+            // a received message is deleted already: nothing to settle, so 
the client stops receiving at once
             assertThat(deferred).isTrue();
             verify(client).stop();
         }
     }
 
+    @Test
+    @Timeout(30)
+    void gracefulStopKeepsTheClientOpenUntilTheExchangesInFlightAreSettled() 
throws Exception {
+        
when(configuration.getServiceBusReceiveMode()).thenReturn(ServiceBusReceiveMode.PEEK_LOCK);
+        settlementFailsOnceTheClientIsClosed();
+        ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint, 
processor);
+        consumer.doStart();
+        when(messageContext.getMessage()).thenReturn(message);
+        processMessageCaptor.getValue().accept(messageContext);
+        Exchange exchange = exchangeCaptor.getValue();
+
+        // the graceful shutdown defers the stop of the consumer while its 
exchange is in flight
+        
assertThat(consumer.deferShutdown(ShutdownRunningTask.CompleteCurrentTaskOnly)).isTrue();
+        assertThat(consumer.getPendingExchangesSize()).isEqualTo(1);
+
+        // the exchange completes during the graceful shutdown: its message is 
completed, not delivered again
+        Synchronization synchronization = 
exchange.getExchangeExtension().handoverCompletions().get(0);
+        synchronization.onComplete(exchange);
+        verify(messageContext).complete();
+        assertThat(consumer.getPendingExchangesSize()).isZero();
+
+        // then the shutdown stops the consumer
+        consumer.doStop();
+        verify(client).close();
+    }
+
+    @Test
+    @Timeout(30)
+    void messageDeliveredDuringGracefulStopIsHeldUntilTheConsumerStops() 
throws Exception {
+        
when(configuration.getServiceBusReceiveMode()).thenReturn(ServiceBusReceiveMode.PEEK_LOCK);
+        ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint, 
processor);
+        consumer.doStart();
+        
assertThat(consumer.deferShutdown(ShutdownRunningTask.CompleteCurrentTaskOnly)).isTrue();
+
+        // the open processor client delivers another message: it is neither 
routed nor settled, and the call does
+        // not return, so the client delivers no more messages
+        ServiceBusReceivedMessageContext delivered = mock();
+        Thread pump = new Thread(() -> 
processMessageCaptor.getValue().accept(delivered));
+        pump.setDaemon(true);
+        pump.start();
+        awaitState(pump, Thread.State.WAITING);
+        assertThat(consumer.getPendingExchangesSize()).isZero();
+
+        // the stop closes the client and releases the call (Service Bus 
delivers the message again when its lock
+        // expires)
+        consumer.doStop();
+        pump.join(TimeUnit.SECONDS.toMillis(10));
+        assertThat(pump.isAlive()).isFalse();
+        verify(client).close();
+        verify(processor, never()).process(any(Exchange.class), 
any(AsyncCallback.class));
+        verifyNoInteractions(delivered);
+    }
+
+    @Test
+    @Timeout(30)
+    void consumerStartedAgainAfterGracefulStopRoutesMessages() throws 
Exception {
+        
when(configuration.getServiceBusReceiveMode()).thenReturn(ServiceBusReceiveMode.PEEK_LOCK);
+        ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint, 
processor);
+        consumer.doStart();
+        
assertThat(consumer.deferShutdown(ShutdownRunningTask.CompleteCurrentTaskOnly)).isTrue();
+        consumer.doStop();
+
+        consumer.doStart();
+        when(messageContext.getMessage()).thenReturn(message);
+        processMessageCaptor.getValue().accept(messageContext);
+
+        verify(processor).process(any(Exchange.class), 
any(AsyncCallback.class));
+        assertThat(consumer.getPendingExchangesSize()).isEqualTo(1);
+        consumer.doStop();
+    }
+
+    /**
+     * As the processor client of azure-messaging-servicebus 7.17 (the default 
V2 stack): stop() closes the processor
+     * like close(), which disposes its receiver (ServiceBusProcessor.stop 
calls close, and the cleanup of its message
+     * pump closes the ServiceBusReceiverAsyncClient), and a settlement 
through a disposed receiver fails with this
+     * IllegalStateException (ServiceBusReceiverAsyncClient.updateDisposition).
+     */
+    private void settlementFailsOnceTheClientIsClosed() {
+        AtomicBoolean closed = new AtomicBoolean();
+        doAnswer(invocation -> {
+            closed.set(true);
+            return null;
+        }).when(client).stop();
+        doAnswer(invocation -> {
+            closed.set(true);
+            return null;
+        }).when(client).close();
+        doAnswer(invocation -> {
+            if (closed.get()) {
+                throw new IllegalStateException("Cannot perform operation 
'completed' on a disposed receiver.");
+            }
+            return null;
+        }).when(messageContext).complete();
+    }
+
+    private static void awaitState(Thread thread, Thread.State state) {
+        await("thread " + thread.getName() + " in state " + state).atMost(10, 
TimeUnit.SECONDS)
+                .until(() -> thread.getState() == state);
+    }
+
     @Test
     void pendingExchangesSizeTracksInflightMessages() throws Exception {
         try (ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint, 
processor)) {

Reply via email to