This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 1eb59175e600 CAMEL-26992: camel-mongodb - a change stream that the
consumer opens again before any exchange completed starts at the current time
and skips the changes made in between (#30136)
1eb59175e600 is described below
commit 1eb59175e60058bbeeedeb0e773ae4e5d7a91ecf
Author: allthingssecurity <[email protected]>
AuthorDate: Mon Nov 16 19:00:22 2026 +0530
CAMEL-26992: camel-mongodb - a change stream that the consumer opens again
before any exchange completed starts at the current time and skips the changes
made in between (#30136)
Found by the DST world in #30135 (CAMEL-26973), which leaves this case out
of its default set: with `-Ddst.mongodb.strictFirstCursor=true`, 50 of 2000
seeds (first failing seed 12) end with changes the route never got. Reproduced
here with a plain unit test, outside the DST harness.
Without a resume token (no `changeStreamToken`, nothing in the
`changeStreamTokenRepository`, no resume strategy) the change streams consumer
opens its change stream without a resume option, so it starts at the current
time. That is right for the first cursor of a route start. But
`MongoDbChangeStreamsThread` also opens a new cursor itself when the change
stream fails, for example when a getMore fails and the resume of the driver
fails too because the server cannot be reached for long [...]
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../mongodb/MongoDbChangeStreamsThread.java | 106 +++++-
.../MongoDbChangeStreamsInvalidateTest.java | 142 ++------
.../mongodb/MongoDbChangeStreamsMocks.java | 142 ++++++++
.../mongodb/MongoDbChangeStreamsReopenTest.java | 361 +++++++++++++++++++++
4 files changed, 643 insertions(+), 108 deletions(-)
diff --git
a/components/camel-mongodb/src/main/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsThread.java
b/components/camel-mongodb/src/main/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsThread.java
index 0c39e45b2846..02a755193ba4 100644
---
a/components/camel-mongodb/src/main/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsThread.java
+++
b/components/camel-mongodb/src/main/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsThread.java
@@ -20,6 +20,7 @@ import java.util.List;
import com.mongodb.MongoCommandException;
import com.mongodb.MongoException;
+import com.mongodb.MongoServerException;
import com.mongodb.client.ChangeStreamIterable;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.model.changestream.ChangeStreamDocument;
@@ -27,6 +28,8 @@ import com.mongodb.client.model.changestream.OperationType;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
import org.bson.BsonDocument;
+import org.bson.BsonInt32;
+import org.bson.BsonTimestamp;
import org.bson.Document;
import org.bson.codecs.DecoderContext;
import org.bson.codecs.DocumentCodec;
@@ -39,9 +42,26 @@ class MongoDbChangeStreamsThread extends
MongoAbstractConsumerThread {
private static final DocumentCodec DOCUMENT_CODEC = new DocumentCodec();
// the server error InvalidResumeToken
private static final int INVALID_RESUME_TOKEN = 260;
+ // the server errors for a start that the oplog no longer has:
ChangeStreamHistoryLost (MongoDB 4.4, and the later
+ // 4.2 releases), ChangeStreamFatalError and 40576 (earlier 4.2 releases)
+ private static final int CHANGE_STREAM_HISTORY_LOST = 286;
+ private static final int CHANGE_STREAM_FATAL_ERROR = 280;
+ private static final int RESUME_POINT_NOT_IN_OPLOG = 40576;
+ // the server errors for a startAtOperationTime that the server does not
accept (MongoDB-compatible servers may
+ // answer so): BadValue, FailedToParse, and 40415 (an unknown field of the
$changeStream stage)
+ private static final int BAD_VALUE = 2;
+ private static final int FAILED_TO_PARSE = 9;
+ private static final int UNKNOWN_FIELD = 40415;
private List<BsonDocument> bsonFilter;
private BsonDocument resumeToken;
+ // the operation time read just before the change stream was opened
without a resume token. Until an exchange
+ // completed (and so there is a resume token), a cursor opened again
starts at it instead of the current time, so
+ // the changes made in between are not skipped. It is not stored: a route
started again without a stored token
+ // still starts at the current time
+ private BsonTimestamp startTime;
+ // whether the server refused startAtOperationTime: the consumer then no
longer tries it
+ private boolean startTimeRefused;
private CommitManager commitManager;
private String resumeTokenKey;
@@ -68,7 +88,7 @@ class MongoDbChangeStreamsThread extends
MongoAbstractConsumerThread {
@Override
protected MongoCursor initializeCursor() {
if (resumeToken == null) {
- return watch().iterator();
+ return openWithoutToken();
}
try {
return watch().resumeAfter(resumeToken).iterator();
@@ -95,6 +115,82 @@ class MongoDbChangeStreamsThread extends
MongoAbstractConsumerThread {
}
}
+ private MongoCursor openWithoutToken() {
+ if (startTime != null) {
+ // no exchange completed since the change stream was first opened:
start where it started
+ try {
+ return watch().startAtOperationTime(startTime).iterator();
+ } catch (MongoCommandException e) {
+ int code = e.getErrorCode();
+ if (isStartNotInOplog(code)) {
+ // the oplog no longer goes back that far (no exchange
completed for longer than the oplog window):
+ // go on from the current time, as the consumer did before
it remembered the start
+ warnStartedAtCurrentTime(e.getErrorMessage());
+ } else if (code == BAD_VALUE || code == FAILED_TO_PARSE ||
code == UNKNOWN_FIELD) {
+ // the server does not accept the option: go on from the
current time, as the consumer did before
+ // it remembered the start, and do not try it again
+ log.warn("The change stream of collection {} cannot be
started at an operation time, as the server"
+ + " does not accept startAtOperationTime ({}).
Started a new change stream at the current"
+ + " time, changes made since the previous one was
opened may not have been consumed.",
+ endpoint.getCollection(), e.getErrorMessage());
+ startTimeRefused = true;
+ } else {
+ throw e;
+ }
+ startTime = null;
+ }
+ }
+ startTime = startTimeRefused ? null : nextOperationTime();
+ return watch().iterator();
+ }
+
+ private static boolean isStartNotInOplog(int code) {
+ return code == CHANGE_STREAM_HISTORY_LOST || code ==
CHANGE_STREAM_FATAL_ERROR || code == RESUME_POINT_NOT_IN_OPLOG;
+ }
+
+ private void warnStartedAtCurrentTime(String error) {
+ log.warn("The change stream of collection {} cannot be started again
where it was first opened, as the oplog no"
+ + " longer goes back that far ({}). Started a new change
stream at the current time, changes made"
+ + " since then may not have been consumed.",
+ endpoint.getCollection(), error);
+ }
+
+ /**
+ * The operation time right after the one in the reply of a {@code ping}
sent just before the change stream is
+ * opened at the current time, or {@code null} when the server does not
report one (a standalone server, which has
+ * no change streams) or answers the ping with an error.
+ * <p>
+ * A change stream without a resume option starts at the current time. The
driver keeps the operation time of the
+ * reply that opened it, to resume the change stream itself, but does not
expose it, and the resume token of the
+ * cursor is only known once a getMore succeeded. So this is where a
cursor that the consumer thread opens again
+ * before any exchange completed starts. When the ping and the change
stream go to the same member it is not later
+ * than where the first change stream started, so a cursor opened again
may also deliver a change made between the
+ * ping and the opening of the first one. With a secondary read preference
they can go to different members, and a
+ * cursor opened again then starts at most at a change made before the
ping.
+ */
+ private BsonTimestamp nextOperationTime() {
+ BsonDocument reply;
+ try {
+ BsonDocument ping = new BsonDocument("ping", new BsonInt32(1));
+ // sent to a server the change stream may read from (runCommand
alone selects the primary)
+ reply = endpoint.getMongoDatabase().runCommand(ping,
dbCol.getReadPreference(), BsonDocument.class);
+ } catch (MongoServerException e) {
+ // the change stream can be opened all the same, only it starts at
the current time if it has to be opened
+ // again. No server to send it to is not caught here: opening the
change stream, with the same read
+ // preference, would fail the same way
+ log.debug("Cannot read the operation time of the server: {}",
e.getMessage());
+ return null;
+ }
+ if (reply == null || !reply.isTimestamp("operationTime")) {
+ return null;
+ }
+ BsonTimestamp operationTime = reply.getTimestamp("operationTime");
+ // for read concern local the operation time of a command that writes
nothing is the timestamp of the newest
+ // entry of the oplog (runCommand reference; the cluster time through
mongos), and a change stream includes the
+ // changes at its startAtOperationTime: start right after it
+ return new BsonTimestamp(operationTime.getTime(),
operationTime.getInc() + 1);
+ }
+
private ChangeStreamIterable<Document> watch() {
ChangeStreamIterable<Document> iterable = bsonFilter != null
? dbCol.watch(bsonFilter)
@@ -156,7 +252,13 @@ class MongoDbChangeStreamsThread extends
MongoAbstractConsumerThread {
// cursor.hasNext() opens socket and waiting for data
// it throws exception when cursor is closed in another thread
// there is no way to stop hasNext() before closing cursor
- if (keepRunning) {
+ if (keepRunning && resumeToken == null && startTime != null &&
isStartNotInOplog(e.getCode())) {
+ // a change stream started at the start time and failed on a
getMore because the oplog no longer goes
+ // back that far (should the server not have refused it in the
aggregate): the next one starts at the
+ // current time, as in openWithoutToken
+ warnStartedAtCurrentTime(e.getMessage());
+ startTime = null;
+ } else if (keepRunning) {
log.debug("Exception from MongoDB, will regenerate cursor.",
e);
} else {
throw e;
diff --git
a/components/camel-mongodb/src/test/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsInvalidateTest.java
b/components/camel-mongodb/src/test/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsInvalidateTest.java
index 8a21b27dbf03..56e7f2d7cc48 100644
---
a/components/camel-mongodb/src/test/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsInvalidateTest.java
+++
b/components/camel-mongodb/src/test/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsInvalidateTest.java
@@ -18,39 +18,36 @@ package org.apache.camel.component.mongodb;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.mongodb.MongoCommandException;
-import com.mongodb.ServerAddress;
-import com.mongodb.client.ChangeStreamIterable;
-import com.mongodb.client.ListCollectionNamesIterable;
import com.mongodb.client.MongoChangeStreamCursor;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoCollection;
-import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.changestream.ChangeStreamDocument;
-import com.mongodb.client.model.changestream.FullDocument;
import com.mongodb.client.model.changestream.OperationType;
import org.apache.camel.CamelContext;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.component.mock.MockEndpoint;
+import
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.ResumeOptions;
import org.apache.camel.impl.DefaultCamelContext;
import org.apache.camel.support.processor.state.MemoryStateRepository;
import org.bson.BsonDocument;
-import org.bson.BsonInt32;
import org.bson.BsonString;
import org.bson.Document;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import static
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.commandFailed;
+import static
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.cursor;
+import static
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.event;
+import static
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.mockCollection;
+import static
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.mockDatabase;
import static org.awaitility.Awaitility.await;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
/**
* A change stream ends with an {@code invalidate} event when its collection
is dropped or renamed, or its database
@@ -59,8 +56,6 @@ import static org.mockito.Mockito.when;
*/
class MongoDbChangeStreamsInvalidateTest {
- private static final ServerAddress SERVER = new ServerAddress("localhost",
27017);
-
private static final BsonDocument INSERT_TOKEN = token("insert");
private static final BsonDocument DROP_TOKEN = token("drop");
private static final BsonDocument INVALIDATE_TOKEN = token("invalidate");
@@ -72,25 +67,13 @@ class MongoDbChangeStreamsInvalidateTest {
private final List<String> opened = Collections.synchronizedList(new
ArrayList<>());
private CamelContext context;
- @SuppressWarnings("unchecked")
- private static MongoCollection<Document> mockCollection() {
- return mock(MongoCollection.class);
- }
-
private static BsonDocument token(String name) {
return new BsonDocument("_data", new BsonString(name));
}
@BeforeEach
void setUp() {
- MongoDatabase database = mock(MongoDatabase.class);
- ListCollectionNamesIterable names =
mock(ListCollectionNamesIterable.class);
- when(names.spliterator()).thenAnswer(i ->
List.of("events").spliterator());
- when(database.listCollectionNames()).thenReturn(names);
- when(database.getCollection("events",
Document.class)).thenReturn(collection);
- when(database.getName()).thenReturn("test");
- when(client.getDatabase("test")).thenReturn(database);
- when(collection.watch()).thenAnswer(i -> new
FakeChangeStream().iterable);
+ mockDatabase(client, collection, this::open);
context = new DefaultCamelContext();
context.getRegistry().bind("myDb", client);
@@ -160,23 +143,10 @@ class MongoDbChangeStreamsInvalidateTest {
return mock;
}
- @SuppressWarnings("unchecked")
- private static ChangeStreamDocument<Document> event(
- OperationType type, BsonDocument token, Document fullDocument) {
- ChangeStreamDocument<Document> event =
mock(ChangeStreamDocument.class);
- when(event.getOperationType()).thenReturn(type);
- when(event.getResumeToken()).thenReturn(token);
- when(event.getFullDocument()).thenReturn(fullDocument);
- if (fullDocument != null) {
- when(event.getDocumentKey()).thenReturn(new BsonDocument("_id",
new BsonInt32(fullDocument.getInteger("_id"))));
- }
- return event;
- }
-
/**
- * One change stream opened with {@code collection.watch()}, answering
like a MongoDB server (4.2 or later) on which
- * the collection was dropped and then created again (change streams
manual, "Invalidate Events" and "Resume a
- * Change Stream"):
+ * Opens the cursor of one change stream ({@code collection.watch()} with
the given resume options), answering like
+ * a MongoDB server (4.2 or later) on which the collection was dropped and
then created again (change streams
+ * manual, "Invalidate Events" and "Resume a Change Stream"):
* <ul>
* <li>without a resume option: the insert, the drop and the invalidate
event; the server then closes the cursor, so
* {@code hasNext()} returns false;</li>
@@ -190,73 +160,33 @@ class MongoDbChangeStreamsInvalidateTest {
* specify both 'resumeAfter' and 'startAfter' in a $changeStream stage"
(the consumer never sets both).</li>
* </ul>
*/
- private final class FakeChangeStream {
-
- private final ChangeStreamIterable<Document> iterable = mockIterable();
- private BsonDocument resumeAfter;
- private BsonDocument startAfter;
-
- @SuppressWarnings("unchecked")
- private static ChangeStreamIterable<Document> mockIterable() {
- return mock(ChangeStreamIterable.class);
- }
-
- private FakeChangeStream() {
-
when(iterable.fullDocument(any(FullDocument.class))).thenReturn(iterable);
- when(iterable.resumeAfter(any(BsonDocument.class))).thenAnswer(i
-> {
- resumeAfter = i.getArgument(0);
- return iterable;
- });
- when(iterable.startAfter(any(BsonDocument.class))).thenAnswer(i ->
{
- startAfter = i.getArgument(0);
- return iterable;
- });
- when(iterable.iterator()).thenAnswer(i -> open());
- }
-
- private MongoChangeStreamCursor<ChangeStreamDocument<Document>> open()
{
- List<ChangeStreamDocument<Document>> events;
- if (resumeAfter != null && startAfter != null) {
- throw commandFailed(50865, "Location50865", "Do not specify
both 'resumeAfter' and 'startAfter' in a"
- + " $changeStream
stage");
- } else if (resumeAfter == null && startAfter == null) {
- opened.add("new");
- events = List.of(
- event(OperationType.INSERT, INSERT_TOKEN, new
Document("_id", 1).append("text", "first")),
- event(OperationType.DROP, DROP_TOKEN, null),
- event(OperationType.INVALIDATE, INVALIDATE_TOKEN,
null));
- } else if (INVALIDATE_TOKEN.equals(resumeAfter)) {
- opened.add("resumeAfter " + resumeAfter);
- throw commandFailed(260, "InvalidResumeToken", "Attempting to
resume a change stream using"
- + "
'resumeAfter' is not allowed from an invalidate"
- + "
notification.");
- } else if (INVALIDATE_TOKEN.equals(startAfter)) {
- opened.add("startAfter " + startAfter);
- events = List.of(event(OperationType.INSERT,
RECREATED_INSERT_TOKEN,
- new Document("_id", 2).append("text", "after the
collection was created again")));
- } else {
- opened.add("resumeAfter " + resumeAfter);
- events = List.of();
- }
- return cursor(events);
- }
-
- @SuppressWarnings("unchecked")
- private MongoChangeStreamCursor<ChangeStreamDocument<Document>> cursor(
- List<ChangeStreamDocument<Document>> events) {
- Iterator<ChangeStreamDocument<Document>> it = events.iterator();
- MongoChangeStreamCursor<ChangeStreamDocument<Document>> cursor =
mock(MongoChangeStreamCursor.class);
- when(cursor.hasNext()).thenAnswer(i -> it.hasNext());
- when(cursor.next()).thenAnswer(i -> it.next());
- return cursor;
- }
-
- private static MongoCommandException commandFailed(int code, String
codeName, String message) {
- return new MongoCommandException(
- new BsonDocument("ok", new BsonInt32(0)).append("code",
new BsonInt32(code))
- .append("codeName", new
BsonString(codeName)).append("errmsg", new BsonString(message)),
- SERVER);
+ private MongoChangeStreamCursor<ChangeStreamDocument<Document>>
open(ResumeOptions options) {
+ BsonDocument resumeAfter = options.resumeAfter;
+ BsonDocument startAfter = options.startAfter;
+ List<ChangeStreamDocument<Document>> events;
+ if (resumeAfter != null && startAfter != null) {
+ throw commandFailed(50865, "Location50865", "Do not specify both
'resumeAfter' and 'startAfter' in a"
+ + " $changeStream
stage");
+ } else if (resumeAfter == null && startAfter == null) {
+ opened.add("new");
+ events = List.of(
+ event(OperationType.INSERT, INSERT_TOKEN, new
Document("_id", 1).append("text", "first")),
+ event(OperationType.DROP, DROP_TOKEN, null),
+ event(OperationType.INVALIDATE, INVALIDATE_TOKEN, null));
+ } else if (INVALIDATE_TOKEN.equals(resumeAfter)) {
+ opened.add("resumeAfter " + resumeAfter);
+ throw commandFailed(260, "InvalidResumeToken", "Attempting to
resume a change stream using"
+ + " 'resumeAfter'
is not allowed from an invalidate"
+ + " notification.");
+ } else if (INVALIDATE_TOKEN.equals(startAfter)) {
+ opened.add("startAfter " + startAfter);
+ events = List.of(event(OperationType.INSERT,
RECREATED_INSERT_TOKEN,
+ new Document("_id", 2).append("text", "after the
collection was created again")));
+ } else {
+ opened.add("resumeAfter " + resumeAfter);
+ events = List.of();
}
+ return cursor(events);
}
}
diff --git
a/components/camel-mongodb/src/test/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsMocks.java
b/components/camel-mongodb/src/test/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsMocks.java
new file mode 100644
index 000000000000..57a35ab97d0c
--- /dev/null
+++
b/components/camel-mongodb/src/test/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsMocks.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.mongodb;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Function;
+
+import com.mongodb.MongoCommandException;
+import com.mongodb.ServerAddress;
+import com.mongodb.client.ChangeStreamIterable;
+import com.mongodb.client.ListCollectionNamesIterable;
+import com.mongodb.client.MongoChangeStreamCursor;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.changestream.ChangeStreamDocument;
+import com.mongodb.client.model.changestream.FullDocument;
+import com.mongodb.client.model.changestream.OperationType;
+import org.bson.BsonDocument;
+import org.bson.BsonInt32;
+import org.bson.BsonString;
+import org.bson.BsonTimestamp;
+import org.bson.Document;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Mocks of the MongoDB driver shared by the tests of the change streams
consumer: the database {@code test} with the
+ * collection {@code events}, a change stream that keeps the resume options it
is given and opens its cursor through the
+ * test, a cursor over a list of events, a change event, and the error of a
command.
+ */
+final class MongoDbChangeStreamsMocks {
+
+ static final ServerAddress SERVER = new ServerAddress("localhost", 27017);
+
+ private MongoDbChangeStreamsMocks() {
+ }
+
+ /** The resume options a change stream was opened with (none, one, or more
when a test sets them). */
+ static final class ResumeOptions {
+ BsonDocument resumeAfter;
+ BsonDocument startAfter;
+ BsonTimestamp startAtOperationTime;
+ }
+
+ @SuppressWarnings("unchecked")
+ static MongoCollection<Document> mockCollection() {
+ return mock(MongoCollection.class);
+ }
+
+ /**
+ * Mocks the database {@code test} of the client with the given collection
as {@code events}, and makes every
+ * {@code collection.watch()} a new change stream whose cursor is opened
by {@code open}, as the aggregate command
+ * does when the consumer calls {@code iterator()}.
+ */
+ static MongoDatabase mockDatabase(
+ MongoClient client, MongoCollection<Document> collection,
+ Function<ResumeOptions,
MongoChangeStreamCursor<ChangeStreamDocument<Document>>> open) {
+ MongoDatabase database = mock(MongoDatabase.class);
+ ListCollectionNamesIterable names =
mock(ListCollectionNamesIterable.class);
+ when(names.spliterator()).thenAnswer(i ->
List.of("events").spliterator());
+ when(database.listCollectionNames()).thenReturn(names);
+ when(database.getCollection("events",
Document.class)).thenReturn(collection);
+ when(database.getName()).thenReturn("test");
+ when(client.getDatabase("test")).thenReturn(database);
+ when(collection.watch()).thenAnswer(i -> changeStream(open));
+ return database;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static ChangeStreamIterable<Document> changeStream(
+ Function<ResumeOptions,
MongoChangeStreamCursor<ChangeStreamDocument<Document>>> open) {
+ ChangeStreamIterable<Document> iterable =
mock(ChangeStreamIterable.class);
+ ResumeOptions options = new ResumeOptions();
+
when(iterable.fullDocument(any(FullDocument.class))).thenReturn(iterable);
+ when(iterable.resumeAfter(any(BsonDocument.class))).thenAnswer(i -> {
+ options.resumeAfter = i.getArgument(0);
+ return iterable;
+ });
+ when(iterable.startAfter(any(BsonDocument.class))).thenAnswer(i -> {
+ options.startAfter = i.getArgument(0);
+ return iterable;
+ });
+
when(iterable.startAtOperationTime(any(BsonTimestamp.class))).thenAnswer(i -> {
+ options.startAtOperationTime = i.getArgument(0);
+ return iterable;
+ });
+ when(iterable.iterator()).thenAnswer(i -> open.apply(options));
+ return iterable;
+ }
+
+ /** A cursor over the given events, whose {@code hasNext()} returns false
after the last one. */
+ @SuppressWarnings("unchecked")
+ static MongoChangeStreamCursor<ChangeStreamDocument<Document>>
cursor(List<ChangeStreamDocument<Document>> events) {
+ Iterator<ChangeStreamDocument<Document>> it = events.iterator();
+ MongoChangeStreamCursor<ChangeStreamDocument<Document>> cursor =
mock(MongoChangeStreamCursor.class);
+ when(cursor.hasNext()).thenAnswer(i -> it.hasNext());
+ when(cursor.next()).thenAnswer(i -> it.next());
+ return cursor;
+ }
+
+ /** A change event; its document key is the {@code _id} of the full
document, when there is one. */
+ @SuppressWarnings("unchecked")
+ static ChangeStreamDocument<Document> event(OperationType type,
BsonDocument token, Document fullDocument) {
+ ChangeStreamDocument<Document> event =
mock(ChangeStreamDocument.class);
+ when(event.getOperationType()).thenReturn(type);
+ when(event.getResumeToken()).thenReturn(token);
+ when(event.getFullDocument()).thenReturn(fullDocument);
+ if (fullDocument != null) {
+ when(event.getDocumentKey()).thenReturn(new BsonDocument("_id",
new BsonInt32(fullDocument.getInteger("_id"))));
+ }
+ return event;
+ }
+
+ /** The reply of a command that failed with the given server error. */
+ static BsonDocument errorReply(int code, String codeName, String message) {
+ return new BsonDocument("ok", new BsonInt32(0)).append("code", new
BsonInt32(code))
+ .append("codeName", new BsonString(codeName)).append("errmsg",
new BsonString(message));
+ }
+
+ /** The exception the driver throws for a command that failed with the
given server error. */
+ static MongoCommandException commandFailed(int code, String codeName,
String message) {
+ return new MongoCommandException(errorReply(code, codeName, message),
SERVER);
+ }
+}
diff --git
a/components/camel-mongodb/src/test/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsReopenTest.java
b/components/camel-mongodb/src/test/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsReopenTest.java
new file mode 100644
index 000000000000..41ecc2866bec
--- /dev/null
+++
b/components/camel-mongodb/src/test/java/org/apache/camel/component/mongodb/MongoDbChangeStreamsReopenTest.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.mongodb;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.mongodb.MongoCommandException;
+import com.mongodb.MongoNodeIsRecoveringException;
+import com.mongodb.MongoSocketReadException;
+import com.mongodb.ReadPreference;
+import com.mongodb.client.MongoChangeStreamCursor;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.changestream.ChangeStreamDocument;
+import com.mongodb.client.model.changestream.OperationType;
+import org.apache.camel.CamelContext;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.mock.MockEndpoint;
+import
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.ResumeOptions;
+import org.apache.camel.impl.DefaultCamelContext;
+import org.bson.BsonDocument;
+import org.bson.BsonInt32;
+import org.bson.BsonString;
+import org.bson.BsonTimestamp;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.SERVER;
+import static
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.errorReply;
+import static
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.event;
+import static
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.mockCollection;
+import static
org.apache.camel.component.mongodb.MongoDbChangeStreamsMocks.mockDatabase;
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * The consumer thread opens a new cursor when the change stream fails (the
driver could not resume it itself). Before
+ * any exchange completed there is no resume token yet, and a change stream
opened without one starts at the current
+ * time, so the new cursor starts at the operation time read when the first
one was opened instead.
+ */
+class MongoDbChangeStreamsReopenTest {
+
+ private final MongoClient client = mock(MongoClient.class);
+ private final MongoCollection<Document> collection = mockCollection();
+ // the changes of the collection, in order: the change at index i has the
operation time (1, i + 1) and the resume
+ // token of position i + 1
+ private final List<String> oplog = Collections.synchronizedList(new
ArrayList<>());
+ // the resume options of every change stream the consumer opened, in order
+ private final List<String> opened = Collections.synchronizedList(new
ArrayList<>());
+ // what happens on the server when the consumer first calls hasNext on the
first change stream: the getMore then
+ // fails, and so does the resume of the driver
+ private volatile Runnable onFirstHasNext = () -> {
+ };
+ // a change made right after the consumer opened a change stream again
+ private volatile String changeAfterFirstReopen;
+ // the oldest position the oplog still has, and the error of a start
before it
+ private volatile int historyStart;
+ private volatile BsonDocument historyLost;
+ // whether the server accepts the start in the aggregate and fails the
first getMore instead
+ private volatile boolean historyLostOnGetMore;
+ // the error of a server that does not accept startAtOperationTime, when
set
+ private volatile BsonDocument startRefused;
+ // the pings the consumer sent to read the operation time
+ private final AtomicInteger pings = new AtomicInteger();
+ // whether a reply carries the operation time (replica sets and sharded
clusters)
+ private volatile boolean operationTimeInReplies = true;
+ // whether the server answers the ping with an error (a member shutting
down)
+ private volatile boolean pingFails;
+ private CamelContext context;
+
+ private static BsonDocument token(int position) {
+ return new BsonDocument("_data", new
BsonString(Integer.toString(position)));
+ }
+
+ @BeforeEach
+ void setUp() {
+ MongoDatabase database = mockDatabase(client, collection, this::open);
+ // the change stream may read from a secondary, so the ping must go
where the change stream reads from
+
when(collection.getReadPreference()).thenReturn(ReadPreference.secondaryPreferred());
+ // runCommand reference: for read concern local, the operation time of
a command that writes nothing is the
+ // timestamp of the most recent entry in the oplog
+ when(database.runCommand(any(Bson.class),
eq(ReadPreference.secondaryPreferred()), eq(BsonDocument.class)))
+ .thenAnswer(i -> {
+ pings.incrementAndGet();
+ if (pingFails) {
+ throw new MongoNodeIsRecoveringException(
+ errorReply(91, "ShutdownInProgress", "The
server is in quiesce mode and will shut down"),
+ SERVER);
+ }
+ BsonDocument reply = new BsonDocument("ok", new
BsonInt32(1));
+ if (operationTimeInReplies) {
+ reply.append("operationTime", new BsonTimestamp(1,
oplog.size()));
+ }
+ return reply;
+ });
+
+ context = new DefaultCamelContext();
+ context.getRegistry().bind("myDb", client);
+ }
+
+ @AfterEach
+ void tearDown() {
+ if (context != null) {
+ context.stop();
+ }
+ }
+
+ @Test
+ void
cursorOpenedAgainBeforeAnyExchangeCompletedStartsWhereTheFirstStarted() throws
Exception {
+ // made before the route starts, so not consumed
+ oplog.add("before");
+ onFirstHasNext = () -> {
+ oplog.add("a");
+ oplog.add("b");
+ };
+ changeAfterFirstReopen = "c";
+
+ MockEndpoint mock = startRoute();
+
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
bodies(mock).contains("c"));
+ assertEquals(List.of("a", "b", "c"), bodies(mock));
+ // after an exchange completed, the consumer resumes after its token
as before
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
opened.contains("resumeAfter " + token(4)));
+ assertEquals(List.of("new", "startAtOperationTime 2"), firstOpened(2));
+ }
+
+ @Test
+ void startNoLongerInTheOplogStartsAtTheCurrentTime() throws Exception {
+ assertStartNoLongerInTheOplogStartsAtTheCurrentTime(errorReply(286,
"ChangeStreamHistoryLost",
+ "Resume of change stream was not possible, as the resume point
may no longer be in the oplog."));
+ }
+
+ @Test
+ void startNoLongerInTheOplogOfMongoDb42StartsAtTheCurrentTime() throws
Exception {
+ // the answer of the 4.2 releases between 40576 and
ChangeStreamHistoryLost
+ assertStartNoLongerInTheOplogStartsAtTheCurrentTime(errorReply(280,
"ChangeStreamFatalError",
+ "resume of change stream was not possible, as the resume point
may no longer be in the oplog."));
+ }
+
+ @Test
+ void startNoLongerInTheOplogOfMongoDb420StartsAtTheCurrentTime() throws
Exception {
+ // the answer of the first 4.2 releases
+ assertStartNoLongerInTheOplogStartsAtTheCurrentTime(errorReply(40576,
"Location40576",
+ "resume of change stream was not possible, as the resume point
may no longer be in the oplog."));
+ }
+
+ private void
assertStartNoLongerInTheOplogStartsAtTheCurrentTime(BsonDocument error) throws
Exception {
+ historyLost = error;
+ onFirstHasNext = () -> {
+ oplog.add("a");
+ oplog.add("b");
+ // the oplog no longer has the time at which the first change
stream started
+ historyStart = 2;
+ };
+ changeAfterFirstReopen = "c";
+
+ MockEndpoint mock = startRoute();
+
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
bodies(mock).contains("c"));
+ assertEquals(List.of("c"), bodies(mock));
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
opened.contains("resumeAfter " + token(3)));
+ assertEquals(List.of("new", "startAtOperationTime 1 failed", "new"),
firstOpened(3));
+ }
+
+ @Test
+ void startNoLongerInTheOplogFoundByTheFirstGetMoreStartsAtTheCurrentTime()
throws Exception {
+ // a server that accepts the start in the aggregate and finds in the
first getMore that the oplog no longer has it
+ historyLost = errorReply(286, "ChangeStreamHistoryLost",
+ "Resume of change stream was not possible, as the resume point
may no longer be in the oplog.");
+ historyLostOnGetMore = true;
+ onFirstHasNext = () -> {
+ oplog.add("a");
+ oplog.add("b");
+ historyStart = 2;
+ };
+ changeAfterFirstReopen = "c";
+
+ MockEndpoint mock = startRoute();
+
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
bodies(mock).contains("c"));
+ assertEquals(List.of("c"), bodies(mock));
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
opened.contains("resumeAfter " + token(3)));
+ assertEquals(List.of("new", "startAtOperationTime 1", "new"),
firstOpened(3));
+ }
+
+ @Test
+ void startAtOperationTimeRefusedStartsAtTheCurrentTimeAndIsNotTriedAgain()
throws Exception {
+ // a MongoDB-compatible server that does not know the option
+ startRefused = errorReply(40415, "Location40415",
+ "BSON field '$changeStream.startAtOperationTime' is an unknown
field.");
+ onFirstHasNext = () -> oplog.add("a");
+ changeAfterFirstReopen = "c";
+
+ MockEndpoint mock = startRoute();
+
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
bodies(mock).contains("c"));
+ assertEquals(List.of("c"), bodies(mock));
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
opened.contains("resumeAfter " + token(2)));
+ assertEquals(List.of("new", "startAtOperationTime 1 refused", "new"),
firstOpened(3));
+ // the operation time was read for the first change stream only
+ assertEquals(1, pings.get());
+ }
+
+ @Test
+ void serverWithoutOperationTimeStartsAtTheCurrentTime() throws Exception {
+ operationTimeInReplies = false;
+ onFirstHasNext = () -> oplog.add("a");
+ changeAfterFirstReopen = "c";
+
+ MockEndpoint mock = startRoute();
+
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
bodies(mock).contains("c"));
+ assertEquals(List.of("c"), bodies(mock));
+ assertEquals(List.of("new", "new"), firstOpened(2));
+ }
+
+ @Test
+ void pingAnsweredWithAnErrorStartsAtTheCurrentTime() throws Exception {
+ pingFails = true;
+ onFirstHasNext = () -> oplog.add("a");
+ changeAfterFirstReopen = "c";
+
+ MockEndpoint mock = startRoute();
+
+ await().atMost(10, TimeUnit.SECONDS).until(() ->
bodies(mock).contains("c"));
+ assertEquals(List.of("c"), bodies(mock));
+ assertEquals(List.of("new", "new"), firstOpened(2));
+ }
+
+ private List<String> firstOpened(int count) {
+ synchronized (opened) {
+ // the consumer thread may still be opening change streams
+ return List.copyOf(opened.subList(0, count));
+ }
+ }
+
+ private static List<String> bodies(MockEndpoint mock) {
+ return mock.getExchanges().stream()
+ .map(e -> e.getIn().getBody(Document.class).getString("text"))
+ .toList();
+ }
+
+ private MockEndpoint startRoute() throws Exception {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+
from("mongodb:myDb?database=test&collection=events&consumerType=changeStreams"
+ + "&cursorRegenerationDelay=100")
+ .routeId("mongo").to("mock:result");
+ }
+ });
+ context.start();
+ return context.getEndpoint("mock:result", MockEndpoint.class);
+ }
+
+ /**
+ * Opens the cursor of one change stream ({@code collection.watch()} with
the given resume options), answering like
+ * a MongoDB server (change streams manual, "Resume a Change Stream"):
+ * <ul>
+ * <li>without a resume option it starts at the current time, with {@code
resumeAfter} after the change of the
+ * token, with {@code startAtOperationTime} at the first change at or
after that time; a start the oplog no longer
+ * has fails with ChangeStreamHistoryLost (286), in the aggregate or, with
historyLostOnGetMore, in the first
+ * getMore; a server that does not accept startAtOperationTime fails the
aggregate with startRefused;</li>
+ * <li>{@code hasNext()} returns false when there is no change, so the
consumer opens a new cursor after the
+ * regeneration delay (a real change stream waits).</li>
+ * </ul>
+ */
+ private MongoChangeStreamCursor<ChangeStreamDocument<Document>>
open(ResumeOptions options) {
+ int position;
+ if (options.resumeAfter != null) {
+ position =
Integer.parseInt(options.resumeAfter.getString("_data").getValue());
+ opened.add("resumeAfter " + options.resumeAfter);
+ } else if (options.startAtOperationTime != null) {
+ int time = options.startAtOperationTime.getInc();
+ position = time - 1;
+ if (startRefused != null) {
+ opened.add("startAtOperationTime " + time + " refused");
+ throw new MongoCommandException(startRefused, SERVER);
+ }
+ if (position < historyStart && historyLostOnGetMore) {
+ opened.add("startAtOperationTime " + time);
+ return historyLostCursor();
+ }
+ if (position < historyStart) {
+ opened.add("startAtOperationTime " + time + " failed");
+ throw new MongoCommandException(historyLost, SERVER);
+ }
+ opened.add("startAtOperationTime " + time);
+ } else {
+ position = oplog.size();
+ opened.add("new");
+ }
+ boolean first = opened.size() == 1;
+ if (!first && changeAfterFirstReopen != null) {
+ oplog.add(changeAfterFirstReopen);
+ changeAfterFirstReopen = null;
+ }
+ return new FakeCursor(position, first).cursor;
+ }
+
+ @SuppressWarnings("unchecked")
+ private MongoChangeStreamCursor<ChangeStreamDocument<Document>>
historyLostCursor() {
+ MongoChangeStreamCursor<ChangeStreamDocument<Document>> cursor =
mock(MongoChangeStreamCursor.class);
+ when(cursor.hasNext()).thenThrow(new
MongoCommandException(historyLost, SERVER));
+ return cursor;
+ }
+
+ private final class FakeCursor {
+
+ @SuppressWarnings("unchecked")
+ private final MongoChangeStreamCursor<ChangeStreamDocument<Document>>
cursor = mock(MongoChangeStreamCursor.class);
+ private int position;
+ private boolean firstHasNextPending;
+
+ private FakeCursor(int position, boolean first) {
+ this.position = position;
+ this.firstHasNextPending = first;
+ when(cursor.hasNext()).thenAnswer(i -> {
+ if (firstHasNextPending) {
+ firstHasNextPending = false;
+ onFirstHasNext.run();
+ throw new MongoSocketReadException("Prematurely reached
end of stream", SERVER);
+ }
+ return position < oplog.size();
+ });
+ when(cursor.next()).thenAnswer(i -> next());
+ }
+
+ private ChangeStreamDocument<Document> next() {
+ String text = oplog.get(position++);
+ return event(OperationType.INSERT, token(position), new
Document("_id", position).append("text", text));
+ }
+ }
+
+}