github-actions[bot] commented on code in PR #66717:
URL: https://github.com/apache/doris/pull/66717#discussion_r3822471742
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java:
##########
@@ -816,41 +864,63 @@ private void dropPartitionsCache(ExternalTable dorisTable,
}
PartitionValueCacheKey key = new
PartitionValueCacheKey(nameMapping, null);
- HivePartitionValues partitionValues =
partitionValuesEntry.getIfPresent(key);
- if (partitionValues == null) {
- return;
+ if (invalidPartitionCache) {
+ for (String partitionName : partitionNames) {
+ invalidatePartitionCache(nameMapping, partitionName);
+ }
}
- HivePartitionValues copy = partitionValues.copy();
- Map<String, Long> partitionNameToIdMapBefore =
copy.getPartitionNameToIdMap();
- Map<Long, PartitionItem> idToPartitionItemBefore =
copy.getIdToPartitionItem();
- Map<Long, List<String>> partitionValuesMap =
copy.getPartitionValuesMap();
-
- for (String partitionName : partitionNames) {
- if (!partitionNameToIdMapBefore.containsKey(partitionName)) {
- LOG.info("dropPartitionsCache partitionName:[{}] not exist
in table:[{}]",
- partitionName, nameMapping.getFullLocalName());
+ for (int attempt = 0; attempt <
PARTITION_EVENT_REPLACE_MAX_RETRIES; attempt++) {
+ HivePartitionValues current =
partitionValuesEntry.peekIfPresent(key);
+ if (current == null) {
+ // Fence a concurrent miss load that may have read HMS
before this event.
+ partitionValuesEntry.invalidateKey(key);
+ return;
+ }
+ HivePartitionValues copy = current.mutableCopy();
+ Map<String, Long> allNames = copy.getPartitionNameToIdMap();
+ Map<Long, PartitionItem> allItems =
copy.getIdToPartitionItem();
+ Map<Long, List<String>> allValues =
copy.getPartitionValuesMap();
+ boolean changed = false;
+ for (String partitionName : partitionNames) {
+ Long partitionId = allNames.remove(partitionName);
+ if (partitionId == null) {
+ LOG.info("dropPartitionsCache partitionName:[{}] not
exist in table:[{}]",
+ partitionName, nameMapping.getFullLocalName());
+ continue;
+ }
+ allItems.remove(partitionId);
+ allValues.remove(partitionId);
+ copy.removePartitionNamePayload(partitionName);
+ changed = true;
+ }
+ if (!changed) {
+ // See the add-event no-op path: event ordering still has
to win over an older refresh.
+ if (partitionValuesEntry.fenceInFlightLoadIfSame(key,
current)) {
+ return;
+ }
continue;
}
- Long partitionId =
partitionNameToIdMapBefore.remove(partitionName);
- idToPartitionItemBefore.remove(partitionId);
- partitionValuesMap.remove(partitionId);
-
- if (invalidPartitionCache) {
- invalidatePartitionCache(nameMapping, partitionName);
+ preparePartitionValuesForPublication(copy);
+ MetaCacheEntry.ReplaceResult result =
partitionValuesEntry.tryReplace(key, current, copy);
Review Comment:
[P2] Preserve the retained key width across drop events
This replacement uses an alias that compares equal to the key loaded with
the table's immutable partition-type list, but weighted `tryReplace` estimates
the copied value with this null-typed key before replacing the existing
mapping. The fixed estimator therefore charges zero key-list bytes, while the
cache and reservation maps update the existing equal-key nodes and continue
retaining the original key and its type-list backing array. After the first
successful drop event, the reservation is smaller than the graph it governs.
Please pass the actual partition column types here (as the add path does), or
make replacement sizing use the retained key/value width, and add a weighted
wide-key drop-event regression.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
