This is an automated email from the ASF dual-hosted git repository.

chrisdutz pushed a commit to branch fix/node-stop-wake-workers
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit cd96f159edf25d2290c9f3ed101219e5a2797fc5
Author: Christofer Dutz <[email protected]>
AuthorDate: Sat Nov 7 15:26:54 2026 +0100

    [Core] Wake the procedure, WAL, disk-usage and periodical workers on stop
    
    Stopping a standalone node spent seconds waiting for worker threads to 
notice the stop:
    - TimeoutExecutorThread.sendStopSignal() was empty; the thread waited in 
its 20 s queue poll
      until awaitTermination interrupted it after eight join rounds, about 2 s 
for each of the two
      timeout threads. It now queues a stop marker that is due at once.
    - Procedure workers slept 1 s after the stopped scheduler returned null; 
now only while running.
    - The WAL close signal without pending data let the serialize task wait out 
the async fsync
      delay (1 s); it now ends the batch at once (pending data and checkpoints 
are still synced).
    - TableDiskUsageIndex.close() waited for its worker's 1 s queue poll; it 
now wakes the worker
      with a no-op operation.
    - AbstractPeriodicalServiceWithAdvance.stopService() now wakes its loop 
instead of letting it
      wait for the next interval.
    - ConfigNodeShutdownHook logged an ERROR on every stop of a single 
ConfigNode, which has nobody
      to report its shutdown to; it only reports when there are targets.
    
    Also, ThriftService.startService() set the caller's interrupt flag for any 
failure, e.g. a
    client port in use (reported as IllegalAccessException), so the cleanup 
after a failed start ran
    on an interrupted thread. Only an InterruptedException re-sets the flag now.
    
    ProcedureExecutorStopTest: an idle executor stopped and joined in 4.6 s 
before, 0.1 s now.
    Overall, stopping a 1C1D node in one JVM went from about 7 s to 1.1 s.
---
 .../confignode/procedure/ProcedureExecutor.java    |  6 ++-
 .../procedure/TimeoutExecutorThread.java           | 18 ++++++-
 .../confignode/service/ConfigNodeShutdownHook.java |  3 +-
 .../procedure/ProcedureExecutorStopTest.java       | 57 ++++++++++++++++++++++
 .../tableDiskUsageIndex/TableDiskUsageIndex.java   | 15 ++++++
 .../dataregion/wal/buffer/WALBuffer.java           |  8 +--
 .../AbstractPeriodicalServiceWithAdvance.java      |  3 ++
 .../iotdb/commons/service/ThriftService.java       | 10 ++--
 8 files changed, 110 insertions(+), 10 deletions(-)

diff --git 
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
 
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
index 32797ba0c7a..f9a1a06e488 100644
--- 
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
+++ 
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
@@ -1028,7 +1028,11 @@ public class ProcedureExecutor<Env> {
         while (isRunning() && keepAlive(lastUpdated)) {
           Procedure<Env> procedure = workerScheduler.poll(keepAliveTime, 
TimeUnit.MILLISECONDS);
           if (procedure == null) {
-            Thread.sleep(1000);
+            // a stopped scheduler returns null at once; leave right away 
instead of
+            // sleeping another second first
+            if (isRunning()) {
+              Thread.sleep(1000);
+            }
             continue;
           }
           boolean executionAcquired = false;
diff --git 
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
 
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
index 23cfb6c089f..73fe43d2c26 100644
--- 
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
+++ 
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
@@ -39,6 +39,9 @@ public class TimeoutExecutorThread<Env> extends 
StoppableThread {
   private final DelayQueue<ProcedureDelayContainer<Env>> queue = new 
DelayQueue<>();
   private final Set<Procedure<Env>> registeredInternalProcedures = 
ConcurrentHashMap.newKeySet();
 
+  /** Queued by {@link #sendStopSignal()}; due at once, carries no procedure. 
*/
+  private final ProcedureDelayContainer<Env> stopMarker = new 
ProcedureDelayContainer<>(null);
+
   public TimeoutExecutorThread(
       ProcedureExecutor<Env> envProcedureExecutor, ThreadGroup threadGroup, 
String name) {
     super(threadGroup, name);
@@ -78,7 +81,7 @@ public class TimeoutExecutorThread<Env> extends 
StoppableThread {
   public void run() {
     while (executor.isRunning()) {
       ProcedureDelayContainer<Env> delayTask = takeQuietly();
-      if (delayTask == null) {
+      if (delayTask == null || delayTask == stopMarker) {
         continue;
       }
       Procedure<Env> procedure = delayTask.getProcedure();
@@ -110,7 +113,15 @@ public class TimeoutExecutorThread<Env> extends 
StoppableThread {
     }
   }
 
-  public void sendStopSignal() {}
+  @Override
+  public void sendStopSignal() {
+    // wake run() from its queue poll (up to DELAY_QUEUE_TIMEOUT seconds), so 
that it
+    // sees the stopped executor at once. Before, join() waited until 
awaitTermination's eighth
+    // join round interrupted the thread, about 2 s per thread on every stop.
+    if (!queue.contains(stopMarker)) {
+      queue.add(stopMarker);
+    }
+  }
 
   private static class ProcedureDelayContainer<Env> implements Delayed {
 
@@ -143,6 +154,9 @@ public class TimeoutExecutorThread<Env> extends 
StoppableThread {
 
     @Override
     public long getDelay(TimeUnit unit) {
+      if (procedure == null) {
+        return 0; // the stop marker
+      }
       long delay = procedure.getTimeoutTimestamp() - 
System.currentTimeMillis();
       return unit.convert(delay, TimeUnit.MILLISECONDS);
     }
diff --git 
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
 
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
index 2296d54cd02..dfac10a0f87 100644
--- 
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
+++ 
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
@@ -101,7 +101,8 @@ public class ConfigNodeShutdownHook extends Thread {
     // redirect. If no leader is reachable, the new leader will mark this node 
Unknown by
     // heartbeat timeout instead.
     
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Stopped);
-    if (!reportShutdown(localNode, reportTargets)) {
+    // a single ConfigNode has nobody to report to; that is no failure worth 
an ERROR
+    if (!reportTargets.isEmpty() && !reportShutdown(localNode, reportTargets)) 
{
       LOGGER.error(
           
ConfigNodeMessages.REPORTING_CONFIGNODE_SHUTDOWN_FAILED_THE_CLUSTER_WILL_STILL_TAKE_THE);
     }
diff --git 
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ProcedureExecutorStopTest.java
 
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ProcedureExecutorStopTest.java
new file mode 100644
index 00000000000..1d6d98b1356
--- /dev/null
+++ 
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ProcedureExecutorStopTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+import org.apache.iotdb.confignode.procedure.env.TestProcEnv;
+import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
+
+import org.junit.Test;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertTrue;
+
+public class ProcedureExecutorStopTest {
+
+  /**
+   * An idle executor must stop without waiting out the poll intervals of its 
threads: the timeout
+   * and worker-monitor threads used to wait until awaitTermination 
interrupted them (about 2 s
+   * each), and every worker slept another second after its scheduler was 
stopped.
+   */
+  @Test(timeout = 60_000)
+  public void anIdleExecutorStopsWithoutWaitingOutItsPollIntervals() {
+    TestProcEnv env = new TestProcEnv();
+    IProcedureStore store = new NoopProcedureStore();
+    ProcedureExecutor<TestProcEnv> executor = new ProcedureExecutor<>(env, 
store);
+    env.setScheduler(executor.getScheduler());
+    executor.init(4);
+    store.start();
+    executor.startWorkers();
+
+    long start = System.nanoTime();
+    executor.stop();
+    executor.join();
+    long tookMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+
+    store.stop();
+    store.cleanup();
+    assertTrue("stop and join took " + tookMillis + " ms", tookMillis < 1_500);
+  }
+}
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
index ea578316511..d9b872dc346 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
@@ -236,6 +236,8 @@ public class TableDiskUsageIndex {
     }
     try {
       stop = true;
+      // wake the worker from its queue poll (up to 1 s) so that it sees the 
stop flag
+      queue.offer(new WakeUpOperation());
       scheduledExecutorService.shutdown();
       scheduledExecutorService.awaitTermination(5, TimeUnit.SECONDS);
       writerMap.values().forEach(DataRegionTableSizeIndexWriter::close);
@@ -303,6 +305,19 @@ public class TableDiskUsageIndex {
     }
   }
 
+  /** Does nothing; {@link #close()} queues it to wake the worker. */
+  private static class WakeUpOperation extends Operation {
+
+    private WakeUpOperation() {
+      super(null, -1);
+    }
+
+    @Override
+    public void apply(TableDiskUsageIndex tableDiskUsageIndex) {
+      // nothing to do
+    }
+  }
+
   protected static class StartReadOperation extends Operation {
     protected final DataRegion region;
     protected final boolean readTsFileIndex;
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
index eba379f1cc5..d1ef522f4bf 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
@@ -443,12 +443,14 @@ public class WALBuffer extends AbstractWALBuffer {
                 identifier,
                 walEntries.size());
           }
-          boolean dataExists = totalSize > 0;
-          if (dataExists) {
+          // end this batch now. Returning false without data made serialize() 
wait out
+          // the async fsync delay (1 s by default) for entries that can no 
longer come, on every
+          // close. Pending data or checkpoints are synced here instead of 
after that wait.
+          if (totalSize > 0 || !info.checkpoints.isEmpty()) {
             fsyncWorkingBuffer(currentSearchIndex, currentFileStatus, info);
           }
           isClosed = true;
-          return dataExists;
+          return true;
         default:
           return false;
       }
diff --git 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java
 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java
index f37931e97e0..6e186815edf 100644
--- 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java
+++ 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java
@@ -89,6 +89,9 @@ public abstract class AbstractPeriodicalServiceWithAdvance {
 
   public void stopService() {
     allowSubmitListen = false;
+    // wake the loop from its interval wait, so that it ends now and its 
executor can
+    // terminate, instead of only after the next interval
+    advanceExecution();
 
     LOGGER.info(ServiceMessages.SERVICE_STOPPED_SUCCESSFULLY, 
getClass().getSimpleName());
   }
diff --git 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java
 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java
index 75acc8d6f00..0edc1461654 100644
--- 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java
+++ 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java
@@ -118,13 +118,17 @@ public abstract class ThriftService implements IService {
         // sleep 100ms for waiting the rpc server start.
         Thread.sleep(100);
       }
-    } catch (InterruptedException
-        | ClassNotFoundException
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new StartupException(this.getID().getName(), e.getMessage());
+    } catch (ClassNotFoundException
         | IllegalAccessException
         | InstantiationException
         | NoSuchMethodException
         | InvocationTargetException e) {
-      Thread.currentThread().interrupt();
+      // only an interruption re-sets the interrupt flag; a failed start (e.g. 
the port is in use,
+      // reported as IllegalAccessException) must not leave the calling thread 
interrupted, or the
+      // cleanup after it runs on an interrupted thread
       throw new StartupException(this.getID().getName(), e.getMessage());
     }
 

Reply via email to