This is an automated email from the ASF dual-hosted git repository.
oscerd pushed a commit to branch camel-4.18.x
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/camel-4.18.x by this push:
new b24eb75256f9 CAMEL-24382: camel-azure-cosmosdb - process change-feed
batches synchronously so events are not lost on failure (#25638)
b24eb75256f9 is described below
commit b24eb75256f9821074580bf27df875b310158209
Author: Andrea Cosentino <[email protected]>
AuthorDate: Thu Aug 27 06:40:07 2026 +0200
CAMEL-24382: camel-azure-cosmosdb - process change-feed batches
synchronously so events are not lost on failure (#25638)
CAMEL-24382: camel-azure-cosmosdb - process change-feed batches
synchronously so events are not lost on failure (#25617)
CosmosDbConsumer.onEventListener dispatched the exchange fire-and-forget
(getAsyncProcessor().process(exchange, EmptyAsyncCallback.get())) and
returned
immediately. The Azure SDK ChangeFeedProcessor uses a synchronous
handleChanges handler and advances (checkpoints) the lease as soon as that
handler returns - i.e. before the route processed the batch. On a route
failure the lease had already advanced, so the batch was never redelivered
and
the change feed silently lost data (at-most-once).
Process the batch synchronously and rethrow on failure, so the lease is only
checkpointed after successful processing and a failed batch is redelivered
on
the next feed poll (at-least-once). The processing is extracted into a
package-private processBatch(Exchange) seam covered by CosmosDbConsumerTest
(failure rethrows, success returns normally).
(cherry picked from commit 4e94d720ec406cd1cf8ac58ba0205b471d069991)
Signed-off-by: Andrea Cosentino <[email protected]>
Co-authored-by: Claude Opus 4.8 <[email protected]>
---
.../component/azure/cosmosdb/CosmosDbConsumer.java | 27 ++++++--
.../azure/cosmosdb/CosmosDbConsumerTest.java | 73 ++++++++++++++++++++++
2 files changed, 94 insertions(+), 6 deletions(-)
diff --git
a/components/camel-azure/camel-azure-cosmosdb/src/main/java/org/apache/camel/component/azure/cosmosdb/CosmosDbConsumer.java
b/components/camel-azure/camel-azure-cosmosdb/src/main/java/org/apache/camel/component/azure/cosmosdb/CosmosDbConsumer.java
index 054e7ede630e..9815169c8951 100644
---
a/components/camel-azure/camel-azure-cosmosdb/src/main/java/org/apache/camel/component/azure/cosmosdb/CosmosDbConsumer.java
+++
b/components/camel-azure/camel-azure-cosmosdb/src/main/java/org/apache/camel/component/azure/cosmosdb/CosmosDbConsumer.java
@@ -24,18 +24,18 @@ import
com.azure.cosmos.implementation.apachecommons.lang.RandomStringUtils;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
import org.apache.camel.Processor;
+import org.apache.camel.RuntimeCamelException;
import
org.apache.camel.component.azure.cosmosdb.client.CosmosAsyncClientWrapper;
import
org.apache.camel.component.azure.cosmosdb.operations.CosmosDbContainerOperations;
import
org.apache.camel.component.azure.cosmosdb.operations.CosmosDbOperationsBuilder;
import org.apache.camel.spi.Synchronization;
import org.apache.camel.support.DefaultConsumer;
-import org.apache.camel.support.EmptyAsyncCallback;
import org.apache.camel.support.SynchronizationAdapter;
import org.apache.camel.util.ObjectHelper;
public class CosmosDbConsumer extends DefaultConsumer {
- private Synchronization onCompletion;
+ private final Synchronization onCompletion = new ConsumerOnCompletion();
private CosmosAsyncClientWrapper clientWrapper;
private ChangeFeedProcessor changeFeedProcessor;
@@ -47,7 +47,6 @@ public class CosmosDbConsumer extends DefaultConsumer {
protected void doInit() throws Exception {
super.doInit();
this.clientWrapper = new
CosmosAsyncClientWrapper(getEndpoint().getCosmosAsyncClient());
- this.onCompletion = new ConsumerOnCompletion();
this.changeFeedProcessor =
getContainerOperations().captureEventsWithChangeFeed(
getLeaseContainerOperations().getContainer(), getHostName(),
this::onEventListener,
getConfiguration().getChangeFeedProcessorOptions());
@@ -83,12 +82,28 @@ public class CosmosDbConsumer extends DefaultConsumer {
}
private void onEventListener(final List<Map<String, ?>> recordList) {
- final Exchange exchange = createAzureCosmosDbExchange(recordList);
+ processBatch(createAzureCosmosDbExchange(recordList));
+ }
+ // Visible for testing.
+ void processBatch(final Exchange exchange) {
// add exchange callback
exchange.getExchangeExtension().addOnCompletion(onCompletion);
- // use default consumer callback
- getAsyncProcessor().process(exchange, EmptyAsyncCallback.get());
+
+ // Process the batch synchronously: the Azure ChangeFeedProcessor
advances (checkpoints) the lease as
+ // soon as this handler returns, so processing must complete before we
return. On failure we rethrow,
+ // so the lease is not advanced and the batch is redelivered on the
next feed poll (at-least-once)
+ // instead of being silently lost.
+ try {
+ getProcessor().process(exchange);
+ } catch (Exception e) {
+ exchange.setException(e);
+ }
+
+ final Exception cause = exchange.getException();
+ if (cause != null) {
+ throw new RuntimeCamelException("Error processing Azure CosmosDB
change feed batch", cause);
+ }
}
private Exchange createAzureCosmosDbExchange(final List<Map<String, ?>>
recordList) {
diff --git
a/components/camel-azure/camel-azure-cosmosdb/src/test/java/org/apache/camel/component/azure/cosmosdb/CosmosDbConsumerTest.java
b/components/camel-azure/camel-azure-cosmosdb/src/test/java/org/apache/camel/component/azure/cosmosdb/CosmosDbConsumerTest.java
new file mode 100644
index 000000000000..dd303a1ed91c
--- /dev/null
+++
b/components/camel-azure/camel-azure-cosmosdb/src/test/java/org/apache/camel/component/azure/cosmosdb/CosmosDbConsumerTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.azure.cosmosdb;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.RuntimeCamelException;
+import org.apache.camel.support.DefaultExchange;
+import org.apache.camel.test.junit5.CamelTestSupport;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class CosmosDbConsumerTest extends CamelTestSupport {
+
+ private CosmosDbConsumer newConsumer(Processor processor) throws Exception
{
+ Map<String, Object> params = new HashMap<>();
+ params.put("databaseEndpoint", "https://test.com:443";);
+ params.put("accountKey", "myKey");
+ CosmosDbEndpoint endpoint = (CosmosDbEndpoint)
context.getComponent("azure-cosmosdb", CosmosDbComponent.class)
+ .createEndpoint("azure-cosmosdb://mydb/myContainer",
"mydb/myContainer", params);
+ return new CosmosDbConsumer(endpoint, processor);
+ }
+
+ private Exchange batchExchange() {
+ Exchange exchange = new DefaultExchange(context);
+ exchange.getIn().setBody(List.of(Map.of("id", "1")));
+ return exchange;
+ }
+
+ @Test
+ void failedBatchIsRethrownSoTheLeaseIsNotAdvanced() throws Exception {
+ // a failing route must not let the change feed advance the lease: the
batch has to be rethrown so
+ // the Azure ChangeFeedProcessor redelivers it (at-least-once) instead
of silently losing it
+ CosmosDbConsumer consumer = newConsumer(exchange -> {
+ throw new IllegalStateException("boom");
+ });
+
+ RuntimeCamelException thrown
+ = assertThrows(RuntimeCamelException.class, () ->
consumer.processBatch(batchExchange()));
+ assertInstanceOf(IllegalStateException.class, thrown.getCause());
+ }
+
+ @Test
+ void successfulBatchReturnsNormally() throws Exception {
+ boolean[] processed = { false };
+ CosmosDbConsumer consumer = newConsumer(exchange -> processed[0] =
true);
+
+ consumer.processBatch(batchExchange());
+
+ assertTrue(processed[0]);
+ }
+}
