github-actions[bot] commented on code in PR #70708:
URL: https://github.com/apache/doris/pull/70708#discussion_r4615451016
##########
fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoDriverCatalogClient.java:
##########
@@ -177,6 +184,126 @@ public BsonTimestamp selectSnapshot(String database,
String collection, String p
}
}
+ @Override
+ public MongoSplitProbe probeSplits(MongoTableHandle table, String field,
String preference,
+ TMongoSnapshot snapshot, int rows, long timeoutMs, Runnable
checkCancelled) {
+ long deadline = System.nanoTime() +
TimeUnit.MILLISECONDS.toNanos(timeoutMs);
+ String index = null;
+ long bytes = 0;
+ try {
+ var collection = probeDatabase(table.getDatabase(), deadline,
checkCancelled)
+ .getCollection(table.getName());
+ try (MongoCursor<Document> indexes =
collection.listIndexes().batchSize(1)
+ .timeoutMode(TimeoutMode.CURSOR_LIFETIME).iterator()) {
+ int count = 0;
+ while (true) {
+ // hasNext may issue getMore. Cancellation must be checked
before that request,
+ // not only after it returns; the lifetime timeout also
covers all cursor batches.
+ checkProbe(deadline, checkCancelled);
+ if (!indexes.hasNext()) {
+ break;
+ }
+ checkProbe(deadline, checkCancelled);
+ Document value = indexes.next();
+ bytes +=
MongoSplitPlan.encode(value.toBsonDocument()).length;
+ if (++count > 128 || bytes > 65536) {
+ throw new MongoRangePlanner.PlanningLimit();
+ }
+ if (eligibleIndex(value, field)) {
+ index = value.getString("name");
+ break;
+ }
+ }
+ }
+ if (index == null) {
+ return new MongoSplitProbe(List.of(), -1, bytes, false);
+ }
+ Number size = probeDatabase(table.getDatabase(), deadline,
checkCancelled)
+ .runCommand(new Document("collStats",
table.getName()).append("scale", 1))
+ .get("size", Number.class);
+ long logicalBytes = size == null ? -1 : size.longValue();
+ List<BsonValue> samples = new ArrayList<>();
+ // Index-prefix sampling is deliberately bounded, not an unbiased
cardinality estimate.
+ // It can produce skewed ranges, but never narrows coverage or
removes the complement.
+ // Project only qualified fixed-width scalars: a giant
array/string key cannot multiply
+ // the FE's retained memory by sample_rows. The remaining types
become BsonNull hints.
+ Document type = new Document("$type", "$" + field);
+ Document supported = new Document("$in", List.of(type,
List.of("int", "long", "objectId", "date")));
+ Document projection = new Document("_id", 0).append("value",
+ new Document("$cond", List.of(supported, "$" + field, new
Document("$literal", null))));
+ try (ClientSession session =
client.startSession(ClientSessionOptions.builder()
+ .causallyConsistent(false).snapshot(true).build())) {
+ session.setSnapshotTimestamp(new BsonTimestamp((int)
snapshot.getTimestampSeconds(),
+ (int) snapshot.getTimestampIncrement()));
+ var data = probeDatabase(table.getDatabase(), deadline,
checkCancelled)
+ .getCollection(table.getName(), BsonDocument.class)
+
.withReadPreference(properties.readPreferenceForSnapshot(preference));
+ // Negative batchSize requests singleBatch in the pinned
driver. Fixed-width samples
+ // fit well below one BSON reply; no remote cursor survives
cancellation/timeout.
+ // Driver 5.14 otherwise adds snapshot readConcern to
killCursors, which MongoDB rejects.
+ try (MongoCursor<BsonDocument> cursor =
data.find(session).hintString(index)
+ .projection(projection).limit(rows).batchSize(-rows)
+ .timeoutMode(TimeoutMode.CURSOR_LIFETIME).iterator()) {
+ while (true) {
+ // Do not let a per-iteration timeout reset or a late
cancellation start
+ // another remote batch after the optional planning
budget has ended.
+ checkProbe(deadline, checkCancelled);
+ if (!cursor.hasNext()) {
+ break;
+ }
+ checkProbe(deadline, checkCancelled);
+ BsonDocument value = cursor.next();
+ bytes += MongoSplitPlan.encode(value).length;
+ if (samples.size() >= rows || bytes > 1024 * 1024) {
+ throw new MongoRangePlanner.PlanningLimit();
+ }
+ samples.add(value.get("value", BsonNull.VALUE));
+ }
+ }
+ }
+ checkProbe(deadline, checkCancelled);
+ return new MongoSplitProbe(samples, logicalBytes, bytes, true);
+ } catch (MongoException error) {
+ checkCancelled.run();
+ // Only the optional planning deadline can trigger auto fallback.
Permission denial,
+ // expired snapshot history, replacement, and transport errors
must fail the statement.
+ // A thread can resume after its deadline while handling an
earlier BadValue/permission
+ // failure. Elapsed time alone must not disguise that source error
as optional timeout.
+ if (error instanceof MongoExecutionTimeoutException || error
instanceof MongoOperationTimeoutException) {
+ throw new MongoRangePlanner.PlanningLimit();
+ }
+ throw failure(error);
+ }
+ }
+
+ static boolean eligibleIndex(Document index, String field) {
+ Document keys = index.get("key", Document.class);
+ if (keys == null || keys.isEmpty() ||
!field.equals(keys.keySet().iterator().next())
+ || !(keys.get(field) instanceof Number direction) ||
Math.abs(direction.intValue()) != 1
Review Comment:
[P2] Reject compound wildcard indexes before hinting the sample
A legal MongoDB 7 index such as `{split_key:1,"payload.$**":1}` passes this
check: its leading key is numeric and wildcard sparsity is implicit ([explicit
sparse:true is
rejected](https://github.com/mongodb/mongo/blob/r7.0.28/src/mongo/db/catalog/index_catalog_impl.cpp#L717-L731)).
With snapshot/auto and `mongodb_split_field='split_key'`, `probeSplits()`
selects it and issues `find(session).hintString(index)` with an empty filter.
By inspection of [MongoDB 7 wildcard
expansion](https://github.com/mongodb/mongo/blob/r7.0.28/src/mongo/db/query/planner_wildcard_helpers.cpp#L466-L537),
the empty predicate supplies no fields to expand, and the [hinted
planner](https://github.com/mongodb/mongo/blob/r7.0.28/src/mongo/db/query/query_planner.cpp#L1492-L1499)
returns `NoQueryExecutionPlans`. This non-timeout error aborts SQL even under
`auto`; an ordinary B-tree later in `listIndexes` is never considered. Please
reject wildcard components throughout the key pattern before choosing the hint
and cover this compound case; the current test only rejects a leading `$**`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]