This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new fc748543aab branch-4.0: [enhance](multi-catalog) Split metadata scan
ranges per split #60257 (#60356)
fc748543aab is described below
commit fc748543aabe655b605a42a9ff4929955c6ec550
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Feb 2 11:17:24 2026 +0800
branch-4.0: [enhance](multi-catalog) Split metadata scan ranges per split
#60257 (#60356)
Cherry-picked from #60257
Co-authored-by: Socrates <[email protected]>
---
.../doris/datasource/tvf/source/MetadataScanNode.java | 17 ++++++-----------
1 file changed, 6 insertions(+), 11 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/MetadataScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/MetadataScanNode.java
index 77dbd29e5a9..ac34ac817c4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/MetadataScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/MetadataScanNode.java
@@ -91,23 +91,18 @@ public class MetadataScanNode extends ExternalScanNode {
scanRangeLocations.add(locations);
} else {
// need to split ranges to send to backends
- List<Backend> backends =
Lists.newArrayList(backendPolicy.getBackends());
List<String> splits = metaScanRange.getSerializedSplits();
- int numSplitsPerBE = Math.max(1, splits.size() / backends.size());
+ int maxConcurrency =
ConnectContext.get().getSessionVariable().getMaxScannersConcurrency();
+ int targetRanges = backendPolicy.numBackends() * Math.max(1,
maxConcurrency);
+ int splitsPerRange = (int) Math.ceil((double) splits.size() /
targetRanges);
+ for (int from = 0; from < splits.size(); from += splitsPerRange) {
+ int to = Math.min(from + splitsPerRange, splits.size());
+ Backend backend = backendPolicy.getNextBe();
- for (int i = 0; i < backends.size(); i++) {
- int from = i * numSplitsPerBE;
- if (from >= splits.size()) {
- continue; // no splits for this backend
- }
- int to = Math.min((i + 1) * numSplitsPerBE, splits.size());
-
- // set splited task to TMetaScanRange
TMetaScanRange subRange = metaScanRange.deepCopy();
subRange.setSerializedSplits(splits.subList(from, to));
TScanRangeLocation location = new TScanRangeLocation();
- Backend backend = backends.get(i);
location.setBackendId(backend.getId());
location.setServer(new TNetworkAddress(backend.getHost(),
backend.getBePort()));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
