This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 5df06208bca8 CAMEL-26970: camel-azure-servicebus - an exchange that
completes during a graceful stop cannot settle its message: deferShutdown
closes the receiver (#30131)
5df06208bca8 is described below
commit 5df06208bca8b7bca002193f02ad7ab4a25938f2
Author: allthingssecurity <[email protected]>
AuthorDate: Mon Nov 16 18:59:19 2026 +0530
CAMEL-26970: camel-azure-servicebus - an exchange that completes during a
graceful stop cannot settle its message: deferShutdown closes the receiver
(#30131)
Follow-up of CAMEL-23260. Found by the Service Bus DST world in #30130
(CAMEL-26956; this PR pairs with it): its default set accepts this case, and
`-Ddst.servicebus.strictClose=true` fails 96 of 1000 seeds on main. Confirmed
outside the DST harness against the SDK classes and with a plain unit test.
CAMEL-23260 made `ServiceBusConsumer` `ShutdownAware` so that the exchanges
in flight at a graceful stop can still settle their message. Its
`deferShutdown` calls `client.stop()`, with the comment "stop accepting new
messages but keep the connection open so that in-flight exchanges can still
complete/abandon messages". With azure-messaging-servicebus 7.17 (javap of
7.17.16) that is not what `stop()` does:
- `ServiceBusProcessorClient.stop()` calls `ServiceBusProcessor.stop()`
when the processor runs on the V2 stack;
- the V2 stack is the default for a non-session processor
(`V2StackSupport.isNonSessionAsyncReceiveEnabled` is true unless
`com.azure.messaging.servicebus.nonSession.asyncReceive.v2` opts out);
- `ServiceBusProcessor.stop()` is `close()`: it disposes the
`RollingMessagePump`, a `Mono.using` whose cleanup closes its
`ServiceBusReceiverAsyncClient`;
- a settlement through a disposed receiver fails: `updateDisposition`
returns `IllegalStateException` "Cannot perform operation '...' on a disposed
receiver".
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../camel-azure/camel-azure-servicebus/pom.xml | 6 +
.../azure/servicebus/ServiceBusConsumer.java | 66 +++++++++--
.../azure/servicebus/ServiceBusConsumerTest.java | 121 ++++++++++++++++++++-
3 files changed, 182 insertions(+), 11 deletions(-)
diff --git a/components/camel-azure/camel-azure-servicebus/pom.xml
b/components/camel-azure/camel-azure-servicebus/pom.xml
index 4e584cec43cb..9687ce9b3207 100644
--- a/components/camel-azure/camel-azure-servicebus/pom.xml
+++ b/components/camel-azure/camel-azure-servicebus/pom.xml
@@ -75,6 +75,12 @@
<version>${mockito-version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <version>${awaitility-version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.camel</groupId>
<artifactId>camel-test-infra-core</artifactId>
diff --git
a/components/camel-azure/camel-azure-servicebus/src/main/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumer.java
b/components/camel-azure/camel-azure-servicebus/src/main/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumer.java
index c2c5a96c4949..147f9da3902f 100644
---
a/components/camel-azure/camel-azure-servicebus/src/main/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumer.java
+++
b/components/camel-azure/camel-azure-servicebus/src/main/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumer.java
@@ -22,6 +22,7 @@ import java.time.OffsetDateTime;
import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@@ -70,6 +71,11 @@ public class ServiceBusConsumer extends DefaultConsumer
implements ShutdownAware
private ServiceBusReceiverAsyncClient renewalClient;
private LockRenewer lockRenewer;
private final AtomicInteger pendingExchanges = new AtomicInteger();
+ // Set by deferShutdown in peek-lock mode: a graceful stop waits for the
exchanges in flight, which settle their
+ // message through the processor client, so the client stays open and the
messages it delivers meanwhile are held
+ // (not routed) until the consumer stops
+ private volatile boolean deferredStop;
+ private volatile CountDownLatch stopped = new CountDownLatch(1);
public ServiceBusConsumer(final ServiceBusEndpoint endpoint, final
Processor processor) {
super(endpoint, processor);
@@ -85,6 +91,9 @@ public class ServiceBusConsumer extends DefaultConsumer
implements ShutdownAware
protected void doStart() throws Exception {
super.doStart();
+ deferredStop = false;
+ stopped = new CountDownLatch(1);
+
LOG.debug("Creating connection to Azure ServiceBus");
// close any previously created client (e.g. on route restart after
doStop)
@@ -129,7 +138,16 @@ public class ServiceBusConsumer extends DefaultConsumer
implements ShutdownAware
}
private void processMessage(ServiceBusReceivedMessageContext
messageContext) {
+ // the latch of this run of the consumer, taken before the check, so a
stop and start meanwhile cannot leave the
+ // message waiting for a later stop
+ CountDownLatch stop = stopped;
pendingExchanges.incrementAndGet();
+ // counted before the check: a graceful stop that does not see this
exchange as pending makes it be held here
+ if (deferredStop) {
+ pendingExchanges.decrementAndGet();
+ holdUntilStopped(stop);
+ return;
+ }
try {
final ServiceBusReceivedMessage message =
messageContext.getMessage();
final Exchange exchange = createServiceBusExchange(message);
@@ -156,6 +174,22 @@ public class ServiceBusConsumer extends DefaultConsumer
implements ShutdownAware
}
}
+ /**
+ * Holds a message delivered during a graceful stop, without routing or
settling it: the processor client delivers
+ * the next message only when processMessage returns, so at most
maxConcurrentCalls messages are held (per session
+ * with sessions). The SDK renews their lock while they are held, up to
maxAutoLockRenewDuration: with 0 the lock of
+ * a held message can expire during the graceful wait, and Service Bus
then delivers it again to a free call, where
+ * it is held again (one more delivery count each time). They are released
when the stop has closed the client:
+ * Service Bus delivers a held message again once its lock expires.
+ */
+ private static void holdUntilStopped(CountDownLatch stop) {
+ try {
+ stop.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
private void processError(ServiceBusErrorContext errorContext) {
final Exchange exchange = createServiceBusExchange(errorContext);
@@ -176,14 +210,19 @@ public class ServiceBusConsumer extends DefaultConsumer
implements ShutdownAware
renewalClient.close();
renewalClient = null;
}
- if (client != null) {
- if (clientCloseable) {
- client.close();
- client = null;
- } else {
- // user-provided client: stop accepting messages but don't
close
- client.stop();
+ try {
+ if (client != null) {
+ if (clientCloseable) {
+ client.close();
+ client = null;
+ } else {
+ // user-provided client: stop accepting messages but don't
close
+ client.stop();
+ }
}
+ } finally {
+ // the client is closed or stopped: release the messages held
during a graceful stop
+ stopped.countDown();
}
// shutdown camel consumer
@@ -205,15 +244,22 @@ public class ServiceBusConsumer extends DefaultConsumer
implements ShutdownAware
client.close();
client = null;
}
+ stopped.countDown();
super.doShutdown();
}
@Override
public boolean deferShutdown(ShutdownRunningTask shutdownRunningTask) {
- // stop accepting new messages but keep the connection open
- // so that in-flight exchanges can still complete/abandon messages
- if (client != null) {
+ if (clientCloseable && getConfiguration().getServiceBusReceiveMode()
== ServiceBusReceiveMode.PEEK_LOCK) {
+ // stop routing new messages but keep the client open, so that
in-flight exchanges can still
+ // complete/abandon their message: the stop() of the processor
client closes its receiver (the default
+ // V2 stack of the SDK), and a message cannot be settled through a
closed receiver. If a stop with
+ // abortAfterTimeout gives up, the consumer stays in this state
until the route is stopped and started
+ deferredStop = true;
+ } else if (client != null) {
+ // receive and delete mode: nothing to settle; a processorClient
given by the user: its messages do not go
+ // through processMessage. Stop receiving
client.stop();
}
return true;
diff --git
a/components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumerTest.java
b/components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumerTest.java
index 863134b0f81e..005ecc5ffbc7 100644
---
a/components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumerTest.java
+++
b/components/camel-azure/camel-azure-servicebus/src/test/java/org/apache/camel/component/azure/servicebus/ServiceBusConsumerTest.java
@@ -20,6 +20,8 @@ import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import com.azure.core.amqp.models.AmqpAnnotatedMessage;
@@ -40,12 +42,14 @@ import org.apache.camel.spi.UuidGenerator;
import org.apache.camel.support.DefaultExchange;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.mockito.ArgumentCaptor;
import reactor.core.publisher.Mono;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.*;
@@ -794,17 +798,132 @@ public class ServiceBusConsumerTest {
}
@Test
- void deferShutdownStopsClientAndReturnsTrue() throws Exception {
+ void deferShutdownStopsUserProvidedClient() throws Exception {
+ // its messages go to the callbacks the user built it with, not
through the consumer
+ when(configuration.getProcessorClient()).thenReturn(client);
+
when(configuration.getServiceBusReceiveMode()).thenReturn(ServiceBusReceiveMode.PEEK_LOCK);
+ try (ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint,
processor)) {
+ consumer.doStart();
+
+
assertThat(consumer.deferShutdown(ShutdownRunningTask.CompleteCurrentTaskOnly)).isTrue();
+ verify(client).stop();
+ }
+ }
+
+ @Test
+ void deferShutdownStopsClientInReceiveAndDeleteMode() throws Exception {
+
when(configuration.getServiceBusReceiveMode()).thenReturn(ServiceBusReceiveMode.RECEIVE_AND_DELETE);
try (ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint,
processor)) {
consumer.doStart();
boolean deferred =
consumer.deferShutdown(ShutdownRunningTask.CompleteAllTasks);
+ // a received message is deleted already: nothing to settle, so
the client stops receiving at once
assertThat(deferred).isTrue();
verify(client).stop();
}
}
+ @Test
+ @Timeout(30)
+ void gracefulStopKeepsTheClientOpenUntilTheExchangesInFlightAreSettled()
throws Exception {
+
when(configuration.getServiceBusReceiveMode()).thenReturn(ServiceBusReceiveMode.PEEK_LOCK);
+ settlementFailsOnceTheClientIsClosed();
+ ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint,
processor);
+ consumer.doStart();
+ when(messageContext.getMessage()).thenReturn(message);
+ processMessageCaptor.getValue().accept(messageContext);
+ Exchange exchange = exchangeCaptor.getValue();
+
+ // the graceful shutdown defers the stop of the consumer while its
exchange is in flight
+
assertThat(consumer.deferShutdown(ShutdownRunningTask.CompleteCurrentTaskOnly)).isTrue();
+ assertThat(consumer.getPendingExchangesSize()).isEqualTo(1);
+
+ // the exchange completes during the graceful shutdown: its message is
completed, not delivered again
+ Synchronization synchronization =
exchange.getExchangeExtension().handoverCompletions().get(0);
+ synchronization.onComplete(exchange);
+ verify(messageContext).complete();
+ assertThat(consumer.getPendingExchangesSize()).isZero();
+
+ // then the shutdown stops the consumer
+ consumer.doStop();
+ verify(client).close();
+ }
+
+ @Test
+ @Timeout(30)
+ void messageDeliveredDuringGracefulStopIsHeldUntilTheConsumerStops()
throws Exception {
+
when(configuration.getServiceBusReceiveMode()).thenReturn(ServiceBusReceiveMode.PEEK_LOCK);
+ ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint,
processor);
+ consumer.doStart();
+
assertThat(consumer.deferShutdown(ShutdownRunningTask.CompleteCurrentTaskOnly)).isTrue();
+
+ // the open processor client delivers another message: it is neither
routed nor settled, and the call does
+ // not return, so the client delivers no more messages
+ ServiceBusReceivedMessageContext delivered = mock();
+ Thread pump = new Thread(() ->
processMessageCaptor.getValue().accept(delivered));
+ pump.setDaemon(true);
+ pump.start();
+ awaitState(pump, Thread.State.WAITING);
+ assertThat(consumer.getPendingExchangesSize()).isZero();
+
+ // the stop closes the client and releases the call (Service Bus
delivers the message again when its lock
+ // expires)
+ consumer.doStop();
+ pump.join(TimeUnit.SECONDS.toMillis(10));
+ assertThat(pump.isAlive()).isFalse();
+ verify(client).close();
+ verify(processor, never()).process(any(Exchange.class),
any(AsyncCallback.class));
+ verifyNoInteractions(delivered);
+ }
+
+ @Test
+ @Timeout(30)
+ void consumerStartedAgainAfterGracefulStopRoutesMessages() throws
Exception {
+
when(configuration.getServiceBusReceiveMode()).thenReturn(ServiceBusReceiveMode.PEEK_LOCK);
+ ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint,
processor);
+ consumer.doStart();
+
assertThat(consumer.deferShutdown(ShutdownRunningTask.CompleteCurrentTaskOnly)).isTrue();
+ consumer.doStop();
+
+ consumer.doStart();
+ when(messageContext.getMessage()).thenReturn(message);
+ processMessageCaptor.getValue().accept(messageContext);
+
+ verify(processor).process(any(Exchange.class),
any(AsyncCallback.class));
+ assertThat(consumer.getPendingExchangesSize()).isEqualTo(1);
+ consumer.doStop();
+ }
+
+ /**
+ * As the processor client of azure-messaging-servicebus 7.17 (the default
V2 stack): stop() closes the processor
+ * like close(), which disposes its receiver (ServiceBusProcessor.stop
calls close, and the cleanup of its message
+ * pump closes the ServiceBusReceiverAsyncClient), and a settlement
through a disposed receiver fails with this
+ * IllegalStateException (ServiceBusReceiverAsyncClient.updateDisposition).
+ */
+ private void settlementFailsOnceTheClientIsClosed() {
+ AtomicBoolean closed = new AtomicBoolean();
+ doAnswer(invocation -> {
+ closed.set(true);
+ return null;
+ }).when(client).stop();
+ doAnswer(invocation -> {
+ closed.set(true);
+ return null;
+ }).when(client).close();
+ doAnswer(invocation -> {
+ if (closed.get()) {
+ throw new IllegalStateException("Cannot perform operation
'completed' on a disposed receiver.");
+ }
+ return null;
+ }).when(messageContext).complete();
+ }
+
+ private static void awaitState(Thread thread, Thread.State state) {
+ await("thread " + thread.getName() + " in state " + state).atMost(10,
TimeUnit.SECONDS)
+ .until(() -> thread.getState() == state);
+ }
+
@Test
void pendingExchangesSizeTracksInflightMessages() throws Exception {
try (ServiceBusConsumer consumer = new ServiceBusConsumer(endpoint,
processor)) {