zhangstar333 commented on code in PR #67650:
URL: https://github.com/apache/doris/pull/67650#discussion_r3975080992
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java:
##########
@@ -302,24 +317,26 @@ private Map<Long, LanceFragmentInfo>
getVisibleFragments(LanceTableMetadata meta
return visible;
}
- private List<Split> createFragmentSplits(LanceTableMetadata metadata,
- Map<Long, LanceFragmentInfo> visibleFragments) {
- long targetRows = 1;
- for (LanceFragmentInfo fragment : visibleFragments.values()) {
- targetRows = Math.max(targetRows,
Math.max(fragment.getPhysicalRows(), 1));
- }
-
- // Keep one fragment per split. Use the largest fragment's physical
row count as the
- // normalization baseline for split weights, so backend scheduling
reflects the relative
- // amount of physical data each fragment scans, including rows covered
by deletion metadata.
- List<Split> splits = new ArrayList<>(visibleFragments.size());
- for (LanceFragmentInfo fragment : visibleFragments.values()) {
- LanceSplit split =
LanceSplit.forFragment(metadata.getDatasetUri(), metadata.getVersion(),
- fragment.getId(), fragment.getPhysicalRows());
- split.setTargetSplitSize(targetRows);
- splits.add(split);
+ private List<Split> createNormalFragmentSplits(LanceTableMetadata metadata,
+ Map<Long, LanceFragmentInfo> visibleFragments, int numBackends) {
+ if (plannedFragmentsPerSplit > 0) {
+ // Keep the debug grouping exact, even when it produces fewer
splits than BEs.
+ IndexSegmentSplitPlan plan = new
IndexSegmentSplitPlan(metadata.getDatasetUri(), metadata.getVersion(), 0);
+ plan.addUncoveredFragments(visibleFragments.values(),
plannedFragmentsPerSplit);
+ return plan.buildSplits();
+ }
+ scalarIndexPlan = LanceScalarIndexPlanner.plan(metadata,
lancePushedConjuncts, visibleFragments);
Review Comment:
no need consider old BEs
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
