github-actions[bot] commented on code in PR #68995:
URL: https://github.com/apache/doris/pull/68995#discussion_r4254639327
##########
fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java:
##########
@@ -1983,69 +1976,72 @@ private boolean isConflict(long srcBe, long destBe,
long tabletId, BalanceType b
return true; // Cannot find metadata, treat as conflict
}
- if (balanceType == BalanceType.GLOBAL) {
- return checkGlobalBalanceConflict(srcBe, destBe, tabletMeta,
beToTabletsInParts, beToTabletsInTables);
- } else if (balanceType == BalanceType.TABLE) {
- return checkTableBalanceConflict(srcBe, destBe, tabletMeta,
beToTabletsInParts);
+ if (balanceType == BalanceType.PARTITION) {
+ return false;
}
-
- return false;
- }
-
- private boolean checkGlobalBalanceConflict(long srcBe, long destBe,
TabletMeta tabletMeta,
- ConcurrentHashMap<Long,
- ConcurrentHashMap<Long,
ConcurrentHashMap<Long, Set<Long>>>>
- beToTabletsInParts,
- ConcurrentHashMap<Long,
ConcurrentHashMap<Long, Set<Long>>>
- beToTabletsInTables) {
- long maxBeSize = getTabletSizeInParts(srcBe, tabletMeta,
beToTabletsInParts);
- long minBeSize = getTabletSizeInParts(destBe, tabletMeta,
beToTabletsInParts);
-
- if (minBeSize >= maxBeSize) {
- return true; // Conflict detected
+ Database db =
Env.getCurrentInternalCatalog().getDbNullable(tabletMeta.getDbId());
+ if (db == null) {
+ return true;
}
-
- maxBeSize = getTabletSizeInBes(srcBe, tabletMeta, beToTabletsInTables);
- minBeSize = getTabletSizeInBes(destBe, tabletMeta,
beToTabletsInTables);
-
- return minBeSize >= maxBeSize; // Conflict detected
- }
-
- private boolean checkTableBalanceConflict(long srcBe, long destBe,
TabletMeta tabletMeta,
- ConcurrentHashMap<Long,
- ConcurrentHashMap<Long,
ConcurrentHashMap<Long, Set<Long>>>>
- beToTabletsInParts) {
- long maxBeSize = getTabletSizeInParts(srcBe, tabletMeta,
beToTabletsInParts);
- long minBeSize = getTabletSizeInParts(destBe, tabletMeta,
beToTabletsInParts);
-
- return minBeSize >= maxBeSize; // Conflict detected
- }
-
- private long getTabletSizeInParts(long beId, TabletMeta tabletMeta,
- ConcurrentHashMap<Long,
- ConcurrentHashMap<Long,
ConcurrentHashMap<Long, Set<Long>>>>
- beToTabletsInParts) {
- ConcurrentHashMap<Long, ConcurrentHashMap<Long, Set<Long>>>
indexToTablets
- = beToTabletsInParts.get(tabletMeta.getPartitionId());
- if (indexToTablets == null) {
- return 0;
+ OlapTable table = (OlapTable)
db.getTableNullable(tabletMeta.getTableId());
+ if (table == null) {
+ return true;
}
- ConcurrentHashMap<Long, Set<Long>> beToTablets =
indexToTablets.get(tabletMeta.getIndexId());
- if (beToTablets == null) {
- return 0;
+ table.readLock();
+ try {
+ Partition partition =
table.getPartition(tabletMeta.getPartitionId());
+ MaterializedIndex index = null;
+ if (partition != null) {
+ for (MaterializedIndex visibleIndex
+ :
partition.getMaterializedIndices(IndexExtState.VISIBLE, true)) {
+ if (visibleIndex.getId() == tabletMeta.getIndexId()) {
+ index = visibleIndex;
+ break;
+ }
+ }
+ }
+ if (index == null) {
+ return true;
+ }
+ Map<Long, Integer> indexCounts = new HashMap<>();
+ countRoutes(indexCounts, index, clusterId, variant);
Review Comment:
[P2] Cache index counts instead of recounting every candidate
Every TABLE/GLOBAL attempt now scans the entire index under the table read
lock, including candidates rejected immediately afterward. For example, with
the bucket limit raised, 100 BEs can have a balanced 100,000-tablet index
(1,000 per BE) plus 100 singleton tables still routed to one BE after
expansion. The partition/table phases skip these distributions, but GLOBAL sees
1,100 versus 1,000 tablets and tries 100 candidates with the default budget.
Most candidates belong to the balanced index and are rejected, adding roughly 9
million route visits to the existing roughly 110,000 source-set sampling
visits. The table-count cache cannot help because the index check returns
first. Cache compact per-index counts and maintain/invalidate them with route
changes to preserve the memory saving without repeatedly recounting an
unchanged index.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
