This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new c56d3186464 Fix delayed operator memory release after fragment
termination (#18387) (#19166)
c56d3186464 is described below
commit c56d31864640013811eec7b347804314c4cdf372
Author: Caideyipi <[email protected]>
AuthorDate: Fri Nov 6 18:19:13 2026 +0800
Fix delayed operator memory release after fragment termination (#18387)
(#19166)
* Fix delayed fragment memory release
* Fix final listener delivery when notifications are rejected
(cherry picked from commit 23e1740ee07e77b4ac2cae54def7bbaac98bdfb3)
---
.../iotdb/db/i18n/DataNodeQueryMessages.java | 7 ++
.../iotdb/db/i18n/DataNodeQueryMessages.java | 7 ++
.../fragment/FragmentInstanceContext.java | 6 +-
.../fragment/FragmentInstanceStateMachine.java | 77 +++++++++++++++++++++-
.../plan/planner/LocalExecutionPlanner.java | 47 ++++++++-----
.../fragment/FragmentInstanceStateMachineTest.java | 56 ++++++++++++++++
.../LocalExecutionPlannerOperatorsMemoryTest.java | 34 ++++++++++
7 files changed, 213 insertions(+), 21 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index 4d04dbaf1d2..1f8300c2807 100644
---
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -23,6 +23,13 @@ public final class DataNodeQueryMessages {
private DataNodeQueryMessages() {}
+ public static final String EXCEPTION_STATECHANGELISTENER_IS_NULL_B030E8A0 =
+ "stateChangeListener is null";
+ public static final String
LOG_ERROR_NOTIFYING_STATE_CHANGE_LISTENER_FOR_ARG_00D668A9 =
+ "Error notifying state change listener for {}";
+ public static final String
LOG_RELEASEMEMORY_RELEASE_ARG_CURRENT_REMAINING_MEMORY_ARG_4E7B461E =
+ "[ReleaseMemory] release: {}, current remaining memory: {}";
+
public static final String
EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477 =
"Failed to convert TsFile %s to tablets. Status: %s";
public static final String
EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD =
diff --git
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index e84e99433f5..32e74a0e5b9 100644
---
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -23,6 +23,13 @@ public final class DataNodeQueryMessages {
private DataNodeQueryMessages() {}
+ public static final String EXCEPTION_STATECHANGELISTENER_IS_NULL_B030E8A0 =
+ "stateChangeListener 为空";
+ public static final String
LOG_ERROR_NOTIFYING_STATE_CHANGE_LISTENER_FOR_ARG_00D668A9 =
+ "通知 {} 的状态变更监听器时出错";
+ public static final String
LOG_RELEASEMEMORY_RELEASE_ARG_CURRENT_REMAINING_MEMORY_ARG_4E7B461E =
+ "[ReleaseMemory] 释放:{},当前剩余内存:{}";
+
public static final String
EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_STATUS_ARG_E7C9F477 =
"将 TsFile %s 转换为 tablet 失败。状态:%s";
public static final String
EXCEPTION_FAILED_TO_CONVERT_TSFILE_ARG_TO_TABLETS_ARG_96358FDD =
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
index 102df82d384..3898be8302d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
@@ -514,11 +514,11 @@ public class FragmentInstanceContext extends QueryContext
{
return cachedResult == null ? atomicReference.get() : cachedResult;
}
- // the state change listener is added here in a separate initialize() method
+ // the final-state listener is added here in a separate initialize() method
// instead of the constructor to prevent leaking the "this" reference to
- // another thread, which will cause unsafe publication of this instance.
+ // listener code before this instance has been fully constructed.
private void initialize() {
- stateMachine.addStateChangeListener(this::updateStatsIfDone);
+ stateMachine.addFinalStateChangeListener(this::updateStatsIfDone);
}
private void updateStatsIfDone(FragmentInstanceState newState) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceStateMachine.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceStateMachine.java
index bef977488ec..02a3e3c3516 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceStateMachine.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceStateMachine.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.queryengine.execution.fragment;
+import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
import org.apache.iotdb.db.queryengine.execution.StateMachine;
import
org.apache.iotdb.db.queryengine.execution.StateMachine.StateChangeListener;
@@ -70,6 +71,10 @@ public class FragmentInstanceStateMachine {
private final List<FragmentInstanceFailureListener>
sourceInstanceFailureListeners =
new ArrayList<>();
+ @GuardedBy("finalStateChangeListeners")
+ private final List<StateChangeListener<FragmentInstanceState>>
finalStateChangeListeners =
+ new ArrayList<>();
+
public FragmentInstanceStateMachine(FragmentInstanceId fragmentInstanceId,
Executor executor) {
this.instanceId = requireNonNull(fragmentInstanceId, "fragmentInstanceId
is null");
this.executor = requireNonNull(executor, "executor is null");
@@ -140,7 +145,41 @@ public class FragmentInstanceStateMachine {
requireNonNull(doneState, "doneState is null");
checkArgument(doneState.isDone(), "doneState %s is not a done state",
doneState);
- instanceState.setIf(doneState, currentState -> !currentState.isDone());
+ List<StateChangeListener<FragmentInstanceState>> listeners;
+ RuntimeException stateChangeFailure = null;
+ Error stateChangeError = null;
+ synchronized (finalStateChangeListeners) {
+ boolean stateChanged;
+ try {
+ stateChanged = instanceState.setIf(doneState, currentState ->
!currentState.isDone());
+ } catch (RuntimeException e) {
+ // StateMachine updates its state before submitting asynchronous
notifications. Preserve
+ // the final-state callback guarantee if notification submission is
rejected afterwards.
+ if (!instanceState.get().isDone()) {
+ throw e;
+ }
+ stateChanged = true;
+ stateChangeFailure = e;
+ } catch (Error e) {
+ if (!instanceState.get().isDone()) {
+ throw e;
+ }
+ stateChanged = true;
+ stateChangeError = e;
+ }
+ if (!stateChanged) {
+ return;
+ }
+ listeners = ImmutableList.copyOf(finalStateChangeListeners);
+ finalStateChangeListeners.clear();
+ }
+ listeners.forEach(listener -> fireFinalStateChangeListener(doneState,
listener));
+ if (stateChangeFailure != null) {
+ throw stateChangeFailure;
+ }
+ if (stateChangeError != null) {
+ throw stateChangeError;
+ }
}
/**
@@ -154,6 +193,42 @@ public class FragmentInstanceStateMachine {
instanceState.addStateChangeListener(stateChangeListener);
}
+ /**
+ * Adds a listener that is notified synchronously when this state machine
first reaches a terminal
+ * state. If the state is already terminal, the listener is notified before
this method returns.
+ *
+ * <p>This is intended only for short, non-blocking bookkeeping that must be
completed before the
+ * state transition method returns. Other cleanup should use {@link
#addStateChangeListener}.
+ */
+ public void addFinalStateChangeListener(
+ StateChangeListener<FragmentInstanceState> stateChangeListener) {
+ requireNonNull(
+ stateChangeListener,
DataNodeQueryMessages.EXCEPTION_STATECHANGELISTENER_IS_NULL_B030E8A0);
+
+ FragmentInstanceState currentState;
+ synchronized (finalStateChangeListeners) {
+ currentState = instanceState.get();
+ if (!currentState.isDone()) {
+ finalStateChangeListeners.add(stateChangeListener);
+ return;
+ }
+ }
+ fireFinalStateChangeListener(currentState, stateChangeListener);
+ }
+
+ @SuppressWarnings("squid:S1181")
+ private void fireFinalStateChangeListener(
+ FragmentInstanceState state, StateChangeListener<FragmentInstanceState>
stateChangeListener) {
+ try {
+ stateChangeListener.stateChanged(state);
+ } catch (Throwable t) {
+ LOGGER.error(
+
DataNodeQueryMessages.LOG_ERROR_NOTIFYING_STATE_CHANGE_LISTENER_FOR_ARG_00D668A9,
+ instanceId,
+ t);
+ }
+ }
+
public void addSourceTaskFailureListener(FragmentInstanceFailureListener
listener) {
Map<FragmentInstanceId, Throwable> failures;
synchronized (this) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
index 04dc7099539..5cff59d8431 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
@@ -167,27 +167,40 @@ public class LocalExecutionPlanner {
}
FragmentInstanceStateMachine stateMachine =
instanceContext.getStateMachine();
if (reservedBytes > 0) {
- stateMachine.addStateChangeListener(
- newState -> {
- if (newState.isDone()) {
- try (SetThreadName fragmentInstanceName =
- new
SetThreadName(stateMachine.getFragmentInstanceId().getFullId())) {
- synchronized (this) {
- this.freeMemoryForOperators += reservedBytes;
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(
- "[ReleaseMemory] release: {}, current remaining
memory: {}",
- reservedBytes,
- freeMemoryForOperators);
- }
- }
- }
- }
- });
+ registerOperatorsMemoryRelease(reservedBytes, stateMachine);
}
return reservedBytes;
}
+ private void registerOperatorsMemoryRelease(
+ long reservedBytes, FragmentInstanceStateMachine stateMachine) {
+ // Regular state listeners share a bounded executor and can be delayed by
driver cleanup.
+ // Return this admission reservation in the transition thread so
consecutive fast failures
+ // cannot exhaust the operators memory pool.
+ stateMachine.addFinalStateChangeListener(
+ ignored -> {
+ try (SetThreadName fragmentInstanceName =
+ new
SetThreadName(stateMachine.getFragmentInstanceId().getFullId())) {
+ synchronized (this) {
+ this.freeMemoryForOperators += reservedBytes;
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ DataNodeQueryMessages
+
.LOG_RELEASEMEMORY_RELEASE_ARG_CURRENT_REMAINING_MEMORY_ARG_4E7B461E,
+ reservedBytes,
+ freeMemoryForOperators);
+ }
+ }
+ }
+ });
+ }
+
+ @TestOnly
+ void registerOperatorsMemoryReleaseForTest(
+ long reservedBytes, FragmentInstanceStateMachine stateMachine) {
+ registerOperatorsMemoryRelease(reservedBytes, stateMachine);
+ }
+
/**
* Try to reserve bytes from the operators free-memory pool.
*
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceStateMachineTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceStateMachineTest.java
index d0121e320c1..75e0acbe08f 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceStateMachineTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceStateMachineTest.java
@@ -28,7 +28,11 @@ import org.junit.Test;
import org.mockito.Mockito;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import static org.apache.iotdb.db.queryengine.common.QueryId.MOCK_QUERY_ID;
import static org.junit.Assert.assertEquals;
@@ -117,4 +121,56 @@ public class FragmentInstanceStateMachineTest {
instanceNotificationExecutor.shutdown();
}
}
+
+ @Test
+ public void testFinalStateChangeListenerIsSynchronousAndCalledOnce() {
+ Executor stalledNotificationExecutor = command -> {};
+ FragmentInstanceId instanceId =
+ new FragmentInstanceId(new PlanFragmentId(MOCK_QUERY_ID, 0), "0");
+ FragmentInstanceStateMachine stateMachine =
+ new FragmentInstanceStateMachine(instanceId,
stalledNotificationExecutor);
+ AtomicInteger invocationCount = new AtomicInteger();
+ AtomicReference<FragmentInstanceState> notifiedState = new
AtomicReference<>();
+
+ stateMachine.addFinalStateChangeListener(
+ state -> {
+ invocationCount.incrementAndGet();
+ notifiedState.set(state);
+ });
+ stateMachine.failed(new RuntimeException("Unknown"));
+
+ assertEquals(FragmentInstanceState.FAILED, notifiedState.get());
+ assertEquals(1, invocationCount.get());
+
+ stateMachine.abort();
+ assertEquals(1, invocationCount.get());
+
+ AtomicReference<FragmentInstanceState> lateNotifiedState = new
AtomicReference<>();
+ stateMachine.addFinalStateChangeListener(lateNotifiedState::set);
+ assertEquals(FragmentInstanceState.FAILED, lateNotifiedState.get());
+ }
+
+ @Test
+ public void testFinalStateChangeListenerIsCalledWhenNotificationIsRejected()
{
+ Executor rejectingNotificationExecutor =
+ command -> {
+ throw new RejectedExecutionException();
+ };
+ FragmentInstanceId instanceId =
+ new FragmentInstanceId(new PlanFragmentId(MOCK_QUERY_ID, 0), "0");
+ FragmentInstanceStateMachine stateMachine =
+ new FragmentInstanceStateMachine(instanceId,
rejectingNotificationExecutor);
+ AtomicInteger invocationCount = new AtomicInteger();
+ stateMachine.addFinalStateChangeListener(state ->
invocationCount.incrementAndGet());
+
+ try {
+ stateMachine.failed(new RuntimeException("Unknown"));
+ fail("Expected notification rejection");
+ } catch (RejectedExecutionException e) {
+ // Preserve the existing state-transition failure while guaranteeing
final callbacks run.
+ }
+
+ assertEquals(FragmentInstanceState.FAILED, stateMachine.getState());
+ assertEquals(1, invocationCount.get());
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerOperatorsMemoryTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerOperatorsMemoryTest.java
index 9768d1177d5..cc763763d51 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerOperatorsMemoryTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlannerOperatorsMemoryTest.java
@@ -19,14 +19,21 @@
package org.apache.iotdb.db.queryengine.plan.planner;
+import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
+import org.apache.iotdb.db.queryengine.common.PlanFragmentId;
import org.apache.iotdb.db.queryengine.common.QueryId;
import org.apache.iotdb.db.queryengine.exception.MemoryNotEnoughException;
+import
org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine;
import
org.apache.iotdb.db.queryengine.plan.planner.memory.NotThreadSafeMemoryReservationManager;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
+import java.util.concurrent.Executor;
+
+import static org.apache.iotdb.db.queryengine.common.QueryId.MOCK_QUERY_ID;
+
public class LocalExecutionPlannerOperatorsMemoryTest {
private static final LocalExecutionPlanner PLANNER =
LocalExecutionPlanner.getInstance();
@@ -78,6 +85,33 @@ public class LocalExecutionPlannerOperatorsMemoryTest {
Assert.assertEquals(freeBefore, PLANNER.getFreeMemoryForOperators());
}
+ @Test
+ public void
testEstimatedMemoryIsReleasedWithoutWaitingForAsyncStateListeners() {
+ long request = Math.min(1024L, PLANNER.getFreeMemoryForOperators());
+ if (request <= 0) {
+ return;
+ }
+ long freeBefore = PLANNER.getFreeMemoryForOperators();
+ long reserved = PLANNER.allocateOperatorsMemoryForTest(request, false);
+ Assert.assertEquals(request, reserved);
+ bytesHeldByTest = reserved;
+
+ Executor stalledNotificationExecutor = command -> {};
+ FragmentInstanceStateMachine stateMachine =
+ new FragmentInstanceStateMachine(
+ new FragmentInstanceId(new PlanFragmentId(MOCK_QUERY_ID, 0), "0"),
+ stalledNotificationExecutor);
+ PLANNER.registerOperatorsMemoryReleaseForTest(reserved, stateMachine);
+
+ stateMachine.failed(new RuntimeException("Unknown"));
+
+ long freeAfterFailure = PLANNER.getFreeMemoryForOperators();
+ if (freeAfterFailure == freeBefore) {
+ bytesHeldByTest = 0L;
+ }
+ Assert.assertEquals(freeBefore, freeAfterFailure);
+ }
+
@Test
public void
testMemoryReservationManagerHighestPriorityAllocatesWhenPoolHasRoom() {
long request = Math.min(1024L, PLANNER.getFreeMemoryForOperators());