Gabriel39 commented on code in PR #70708:
URL: https://github.com/apache/doris/pull/70708#discussion_r4640771371


##########
fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoRangePlanner.java:
##########
@@ -0,0 +1,370 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.mongodb;
+
+import org.apache.doris.connector.spi.ConnectorSession;
+import org.apache.doris.connector.spi.ConnectorStatementScope;
+import org.apache.doris.connector.spi.DorisConnectorException;
+import org.apache.doris.connector.spi.scan.ConnectorScanContext;
+import org.apache.doris.thrift.TMongoSnapshot;
+
+import org.bson.BsonValue;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+
+/** Complete planning precedes publication. Statement memos retain either that 
plan or its failure. */
+final class MongoRangePlanner {
+    static final int SAMPLE_ROWS = 4096;
+    static final int PROBE_TIMEOUT_MS = 2000;
+    private final String memoKey = "mongodb.split.plan:" + UUID.randomUUID();
+    private final Supplier<MongoCatalogClient> client;
+
+    MongoRangePlanner(Supplier<MongoCatalogClient> client) {
+        this.client = client;
+    }
+
+    MongoSplitPlan plan(ConnectorSession session, MongoTableHandle table, 
TMongoSnapshot snapshot,
+            String preference, ConnectorScanContext context, long 
descriptorBytes) {
+        checkCancelled(session, context);
+        var options = session.getSessionProperties();
+        String strategy = options.getOrDefault("mongodb_split_strategy", 
"single");
+        if (!Set.of("single", "auto", "range").contains(strategy)) {
+            throw new IllegalArgumentException("Invalid 
mongodb_split_strategy");
+        }
+        if ("single".equals(strategy)) {
+            return MongoSplitPlan.single(1, 1, "single_strategy");
+        }
+        String field = options.getOrDefault("mongodb_split_field", "");
+        boolean custom = !field.isEmpty();
+        if (!custom) {
+            field = "_id";
+        }
+        if (!MongoPredicateTranslator.safePath(field) || field.isEmpty() || 
field.length() > 256) {
+            throw new IllegalArgumentException("MongoDB split field must be an 
unambiguous top-level field");
+        }
+        String selectedField = field;
+        Budget budget = 
session.getStatementScope().computeIfAbsent("mongodb.split.budget", 
Budget::new);
+        String tableKey = tableKey(session, table);
+        // Count fallback scans too: a view must not make a global custom 
field look unambiguous.
+        if (custom) {
+            budget.customField(tableKey);
+        }
+        Memo memo = session.getStatementScope().computeIfAbsent(tableKey, 
Memo::new);
+        return memo.get(session, context, () -> load(session, table, snapshot, 
preference, context,
+                strategy, selectedField, descriptorBytes, budget));
+    }
+
+    Optional<MongoSplitPlan> planned(ConnectorSession session, 
MongoTableHandle table) {
+        Memo memo = session.getStatementScope().getIfPresent(tableKey(session, 
table));
+        return memo == null ? Optional.empty() : memo.completed();
+    }
+
+    private String tableKey(ConnectorSession session, MongoTableHandle table) {
+        // One provider can see several namespaces; never reuse another 
collection's bounds/admission.
+        return memoKey + ":" + session.getCatalogId() + ":" + 
table.getSchema().getVersion();
+    }
+
+    private MongoSplitPlan load(ConnectorSession session, MongoTableHandle 
table, TMongoSnapshot snapshot,
+            String preference, ConnectorScanContext context, String strategy, 
String field,
+            long descriptorBytes, Budget budget) {
+        int maximum = splitLimit(session);
+        targetBytes(session, -1);

Review Comment:
   Addressed in `54ff217bdb74`: added validateSplitSizes(session), named the 8 
MiB descriptor-plan cap, and placed the positive descriptorBytes invariant 
immediately before division, with a comment explaining the single-strategy 
early return. No dependency or user setting was added. FE unit tests, full 
Checkstyle and the FE build pass.



##########
fe/fe-connector/fe-connector-mongodb/src/main/java/org/apache/doris/connector/mongodb/MongoSplitPlan.java:
##########
@@ -0,0 +1,197 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.mongodb;
+
+import org.bson.BsonArray;
+import org.bson.BsonDocument;
+import org.bson.BsonString;
+import org.bson.BsonValue;
+import org.bson.RawBsonDocument;
+import org.bson.codecs.BsonDocumentCodec;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+
+/** Frozen value partitions. Samples affect balance only; the unbounded ends 
and complement ensure coverage. */
+final class MongoSplitPlan {
+    private final String field;
+    private final String family;
+    private final int requested;
+    private final int admitted;
+    private final String reason;
+    private final List<byte[]> partitions;
+    private final long descriptorBudget;
+
+    private MongoSplitPlan(String field, String family, int requested, int 
admitted, String reason,
+            List<BsonDocument> partitions) {
+        this.field = field;
+        this.family = family;
+        this.requested = requested;
+        this.admitted = admitted;
+        this.reason = reason;
+        this.partitions = 
partitions.stream().map(MongoSplitPlan::encode).toList();
+        this.descriptorBudget = 0;
+    }
+
+    private MongoSplitPlan(MongoSplitPlan plan, long descriptorBudget) {
+        this.field = plan.field;
+        this.family = plan.family;
+        this.requested = plan.requested;
+        this.admitted = plan.admitted;
+        this.reason = plan.reason;
+        this.partitions = plan.partitions;
+        this.descriptorBudget = descriptorBudget;
+    }
+
+    MongoSplitPlan withDescriptorBudget(long bytes) {
+        return new MongoSplitPlan(this, bytes);
+    }
+
+    long descriptorBudget() {
+        return descriptorBudget;
+    }
+
+    static MongoSplitPlan single(int requested, int admitted, String reason) {
+        return new MongoSplitPlan("", "", requested, admitted, reason, 
List.of(new BsonDocument()));
+    }
+
+    static MongoSplitPlan ranges(String field, String family, List<BsonValue> 
samples, int requested, int admitted) {
+        if (!Set.of("integer", "date", "objectId").contains(family)) {
+            throw new IllegalArgumentException("Unsupported MongoDB split type 
family");
+        }
+        if (!MongoPredicateTranslator.safePath(field) || field.isEmpty() || 
admitted < 1 || admitted > 256
+                || requested < admitted) {
+            throw new IllegalArgumentException("Invalid MongoDB split plan");
+        }
+        if (admitted == 1 || samples.isEmpty()) {
+            return single(requested, admitted, admitted == 1
+                    ? (requested == 1 ? "requested_one" : "admission_single") 
: "empty_sample");
+        }
+        var values = new ArrayList<>(samples);
+        Comparator<BsonValue> comparator = comparator(family);
+        values.sort(comparator);
+        List<BsonValue> boundaries = new ArrayList<>();
+        // _id always exists and cannot be an array. Expression comparisons 
span the complete BSON
+        // order, including unsampled types; ordinary query comparisons 
instead bracket by type.
+        // Custom fields can be missing or multikey, so retain their typed 
ranges and complement.
+        boolean wholeId = "_id".equals(field);
+        int ranges = wholeId ? admitted : admitted - 1;
+        for (int j = 1; j < ranges; j++) {
+            BsonValue value = values.get((int) ((long) j * values.size() / 
ranges));
+            if (comparator.compare(value, values.get(0)) > 0
+                    && (wholeId || comparator.compare(value, 
values.get(values.size() - 1)) < 0)
+                    && (boundaries.isEmpty() || comparator.compare(value, 
boundaries.get(boundaries.size() - 1)) > 0)) {
+                boundaries.add(value);
+            }
+        }
+        // Without an internal bound, typed ranges plus their complement add a 
full scan
+        // without splitting the useful key space. One complete cursor 
preserves all BSON types.
+        if (boundaries.isEmpty()) {
+            return single(requested, admitted, "insufficient_boundaries");
+        }
+        var types = "integer".equals(family) ? List.of("int", "long") : 
List.of(family);
+        var eligibility = new BsonDocument("$in", new BsonArray(List.of(
+                new BsonDocument("$type", new BsonString("$" + field)),
+                new BsonArray(types.stream().map(BsonString::new).toList()))));
+        List<BsonDocument> partitions = new ArrayList<>();
+        for (int i = 0; i <= boundaries.size(); i++) {
+            if (wholeId) {

Review Comment:
   Expanded the existing replica-set source check to both planning modes and 
every mongos shard in `54ff217bdb74`. It requires real 2..8-split _id reads for 
integer, mixed-type and Date IDs, complete 601-row results, exactly 601 native 
data documents examined, and the original snapshot T. All 24 multi-split 
queries pass on MongoDB 7.0.28/8.0.20 replica sets and mongos. A partition 
changing to COLLSCAN fails the source-work assertion even if SQL results remain 
correct. Replica-set sampling retains its no-getMore assertion; internal mongos 
shard cursors are checked separately from FE sampling.



##########
regression-test/conf/regression-conf.groovy:
##########
@@ -286,7 +286,10 @@ trino_jdbc_port = 8080
 // trino-connector catalog test config
 enableTrinoConnectorTest = false
 
-// Optional MongoDB fixture: run-thirdparties-docker.sh -c mongodb.
+// Native coverage must not depend on installing the optional Trino MongoDB 
plugin.
+// Start the fixture with run-thirdparties-docker.sh -c mongodb.
+enableMongoDBNativeTest = true

Review Comment:
   Fixed in `54ff217bdb74`. Shared/local defaults now disable native, split and 
read-parameter suites. External enables all three after MongoDB startup. The 
local-default selector check executes no MongoDB queries; External defaults 
execute the enabled suites. Missing advanced fixture configuration fails 
explicitly instead of silently skipping.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to