ColinLeeo commented on code in PR #18869:
URL: https://github.com/apache/iotdb/pull/18869#discussion_r4320823669
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java:
##########
@@ -390,66 +472,103 @@ public void free(String queryId, String
fragmentInstanceId, String planNodeId, l
Validate.notNull(queryId,
DataNodeQueryMessages.EXCEPTION_QUERYID_CAN_NOT_BE_NULL_DOT_16639DBE);
Validate.isTrue(bytes > 0L);
- try {
- queryMemoryReservations
- .get(queryId)
- .get(fragmentInstanceId)
- .computeIfPresent(
- planNodeId,
- (k, reservedMemory) -> {
- if (reservedMemory < bytes) {
- throw new IllegalArgumentException(
-
DataNodeQueryMessages.FREE_MORE_MEMORY_THAN_HAS_BEEN_RESERVED);
- }
- return reservedMemory - bytes;
- });
- } catch (NullPointerException e) {
+ Map<String, FragmentMemoryReservation> fragments =
queryMemoryReservations.get(queryId);
+ FragmentMemoryReservation fragment =
+ fragments == null ? null : fragments.get(fragmentInstanceId);
+ if (fragment == null) {
throw new IllegalArgumentException(
DataNodeQueryMessages
-
.QUERY_EXCEPTION_RELATEDMEMORYRESERVED_CAN_T_BE_NULL_WHEN_FREEING_MEMORY_C80009F2,
- e);
+
.QUERY_EXCEPTION_RELATEDMEMORYRESERVED_CAN_T_BE_NULL_WHEN_FREEING_MEMORY_C80009F2);
}
-
- memoryBlock.release(bytes);
+ PlanNodeReservation reservation =
+ requireReservation(
+ fragment.planNodes.get(planNodeId), queryId, fragmentInstanceId,
planNodeId);
+ Object accountingLock = fragment.sharedBudget ? fragment : reservation;
+ synchronized (accountingLock) {
+ if (reservation.bytesReserved < bytes) {
+ throw new IllegalArgumentException(
+ DataNodeQueryMessages.FREE_MORE_MEMORY_THAN_HAS_BEEN_RESERVED);
+ }
+ reservation.bytesReserved -= bytes;
+ fragment.getUsedBytes(reservation.side).addAndGet(-bytes);
+ memoryBlock.release(bytes);
+ }
+ // Complete futures outside the accounting lock: listeners can reserve or
free other inputs.
if (memoryReservationFutures.isEmpty()) {
return;
}
- Iterator<MemoryReservationFuture<Void>> iterator =
memoryReservationFutures.iterator();
+ Iterator<MemoryReservationFuture<Void>> iterator =
memoryReservationFutures.values().iterator();
while (iterator.hasNext()) {
MemoryReservationFuture<Void> future = iterator.next();
- synchronized (future) {
- if (future.isCancelled() || future.isDone()) {
- iterator.remove();
- continue;
- }
- long bytesToReserve = future.getBytesToReserve();
- String curQueryId = future.getQueryId();
- String curFragmentInstanceId = future.getFragmentInstanceId();
- String curPlanNodeId = future.getPlanNodeId();
- long maxBytesCanReserve = future.getMaxBytesCanReserve();
- if (tryReserve(
- curQueryId, curFragmentInstanceId, curPlanNodeId, bytesToReserve,
maxBytesCanReserve)) {
- future.set(null);
- iterator.remove();
- } else {
- rollbackReserve(curQueryId, curFragmentInstanceId, curPlanNodeId,
bytesToReserve);
- }
+ if (tryCompleteReservation(future)) {
+ removePendingReservation(future);
+ }
Review Comment:
Agreed. The local fix protects waiter publication and deregistration with
the same `FragmentMemoryReservation` monitor. Deregistration marks the account
as closed before removing it, and publication refuses requests against a closed
account. Retries encountering a closed account detach the waiter and continue,
without cancelling or completing its future.
Added a deterministic regression reproducing this interleaving and verifying
that another query's waiter is still admitted when memory is freed. There is
also coverage for recreating an FI with the same ID. Both tests fail on
788a995c75 and pass with the local fix. The changes have not been pushed yet.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]