This is an automated email from the ASF dual-hosted git repository. chrisdutz pushed a commit to branch fix/node-stop-wake-workers in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cd96f159edf25d2290c9f3ed101219e5a2797fc5 Author: Christofer Dutz <[email protected]> AuthorDate: Sat Nov 7 15:26:54 2026 +0100 [Core] Wake the procedure, WAL, disk-usage and periodical workers on stop Stopping a standalone node spent seconds waiting for worker threads to notice the stop: - TimeoutExecutorThread.sendStopSignal() was empty; the thread waited in its 20 s queue poll until awaitTermination interrupted it after eight join rounds, about 2 s for each of the two timeout threads. It now queues a stop marker that is due at once. - Procedure workers slept 1 s after the stopped scheduler returned null; now only while running. - The WAL close signal without pending data let the serialize task wait out the async fsync delay (1 s); it now ends the batch at once (pending data and checkpoints are still synced). - TableDiskUsageIndex.close() waited for its worker's 1 s queue poll; it now wakes the worker with a no-op operation. - AbstractPeriodicalServiceWithAdvance.stopService() now wakes its loop instead of letting it wait for the next interval. - ConfigNodeShutdownHook logged an ERROR on every stop of a single ConfigNode, which has nobody to report its shutdown to; it only reports when there are targets. Also, ThriftService.startService() set the caller's interrupt flag for any failure, e.g. a client port in use (reported as IllegalAccessException), so the cleanup after a failed start ran on an interrupted thread. Only an InterruptedException re-sets the flag now. ProcedureExecutorStopTest: an idle executor stopped and joined in 4.6 s before, 0.1 s now. Overall, stopping a 1C1D node in one JVM went from about 7 s to 1.1 s. --- .../confignode/procedure/ProcedureExecutor.java | 6 ++- .../procedure/TimeoutExecutorThread.java | 18 ++++++- .../confignode/service/ConfigNodeShutdownHook.java | 3 +- .../procedure/ProcedureExecutorStopTest.java | 57 ++++++++++++++++++++++ .../tableDiskUsageIndex/TableDiskUsageIndex.java | 15 ++++++ .../dataregion/wal/buffer/WALBuffer.java | 8 +-- .../AbstractPeriodicalServiceWithAdvance.java | 3 ++ .../iotdb/commons/service/ThriftService.java | 10 ++-- 8 files changed, 110 insertions(+), 10 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java index 32797ba0c7a..f9a1a06e488 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java @@ -1028,7 +1028,11 @@ public class ProcedureExecutor<Env> { while (isRunning() && keepAlive(lastUpdated)) { Procedure<Env> procedure = workerScheduler.poll(keepAliveTime, TimeUnit.MILLISECONDS); if (procedure == null) { - Thread.sleep(1000); + // a stopped scheduler returns null at once; leave right away instead of + // sleeping another second first + if (isRunning()) { + Thread.sleep(1000); + } continue; } boolean executionAcquired = false; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java index 23cfb6c089f..73fe43d2c26 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java @@ -39,6 +39,9 @@ public class TimeoutExecutorThread<Env> extends StoppableThread { private final DelayQueue<ProcedureDelayContainer<Env>> queue = new DelayQueue<>(); private final Set<Procedure<Env>> registeredInternalProcedures = ConcurrentHashMap.newKeySet(); + /** Queued by {@link #sendStopSignal()}; due at once, carries no procedure. */ + private final ProcedureDelayContainer<Env> stopMarker = new ProcedureDelayContainer<>(null); + public TimeoutExecutorThread( ProcedureExecutor<Env> envProcedureExecutor, ThreadGroup threadGroup, String name) { super(threadGroup, name); @@ -78,7 +81,7 @@ public class TimeoutExecutorThread<Env> extends StoppableThread { public void run() { while (executor.isRunning()) { ProcedureDelayContainer<Env> delayTask = takeQuietly(); - if (delayTask == null) { + if (delayTask == null || delayTask == stopMarker) { continue; } Procedure<Env> procedure = delayTask.getProcedure(); @@ -110,7 +113,15 @@ public class TimeoutExecutorThread<Env> extends StoppableThread { } } - public void sendStopSignal() {} + @Override + public void sendStopSignal() { + // wake run() from its queue poll (up to DELAY_QUEUE_TIMEOUT seconds), so that it + // sees the stopped executor at once. Before, join() waited until awaitTermination's eighth + // join round interrupted the thread, about 2 s per thread on every stop. + if (!queue.contains(stopMarker)) { + queue.add(stopMarker); + } + } private static class ProcedureDelayContainer<Env> implements Delayed { @@ -143,6 +154,9 @@ public class TimeoutExecutorThread<Env> extends StoppableThread { @Override public long getDelay(TimeUnit unit) { + if (procedure == null) { + return 0; // the stop marker + } long delay = procedure.getTimeoutTimestamp() - System.currentTimeMillis(); return unit.convert(delay, TimeUnit.MILLISECONDS); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java index 2296d54cd02..dfac10a0f87 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java @@ -101,7 +101,8 @@ public class ConfigNodeShutdownHook extends Thread { // redirect. If no leader is reachable, the new leader will mark this node Unknown by // heartbeat timeout instead. CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Stopped); - if (!reportShutdown(localNode, reportTargets)) { + // a single ConfigNode has nobody to report to; that is no failure worth an ERROR + if (!reportTargets.isEmpty() && !reportShutdown(localNode, reportTargets)) { LOGGER.error( ConfigNodeMessages.REPORTING_CONFIGNODE_SHUTDOWN_FAILED_THE_CLUSTER_WILL_STILL_TAKE_THE); } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ProcedureExecutorStopTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ProcedureExecutorStopTest.java new file mode 100644 index 00000000000..1d6d98b1356 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ProcedureExecutorStopTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure; + +import org.apache.iotdb.confignode.procedure.env.TestProcEnv; +import org.apache.iotdb.confignode.procedure.store.IProcedureStore; + +import org.junit.Test; + +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertTrue; + +public class ProcedureExecutorStopTest { + + /** + * An idle executor must stop without waiting out the poll intervals of its threads: the timeout + * and worker-monitor threads used to wait until awaitTermination interrupted them (about 2 s + * each), and every worker slept another second after its scheduler was stopped. + */ + @Test(timeout = 60_000) + public void anIdleExecutorStopsWithoutWaitingOutItsPollIntervals() { + TestProcEnv env = new TestProcEnv(); + IProcedureStore store = new NoopProcedureStore(); + ProcedureExecutor<TestProcEnv> executor = new ProcedureExecutor<>(env, store); + env.setScheduler(executor.getScheduler()); + executor.init(4); + store.start(); + executor.startWorkers(); + + long start = System.nanoTime(); + executor.stop(); + executor.join(); + long tookMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + + store.stop(); + store.cleanup(); + assertTrue("stop and join took " + tookMillis + " ms", tookMillis < 1_500); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java index ea578316511..d9b872dc346 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java @@ -236,6 +236,8 @@ public class TableDiskUsageIndex { } try { stop = true; + // wake the worker from its queue poll (up to 1 s) so that it sees the stop flag + queue.offer(new WakeUpOperation()); scheduledExecutorService.shutdown(); scheduledExecutorService.awaitTermination(5, TimeUnit.SECONDS); writerMap.values().forEach(DataRegionTableSizeIndexWriter::close); @@ -303,6 +305,19 @@ public class TableDiskUsageIndex { } } + /** Does nothing; {@link #close()} queues it to wake the worker. */ + private static class WakeUpOperation extends Operation { + + private WakeUpOperation() { + super(null, -1); + } + + @Override + public void apply(TableDiskUsageIndex tableDiskUsageIndex) { + // nothing to do + } + } + protected static class StartReadOperation extends Operation { protected final DataRegion region; protected final boolean readTsFileIndex; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index eba379f1cc5..d1ef522f4bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -443,12 +443,14 @@ public class WALBuffer extends AbstractWALBuffer { identifier, walEntries.size()); } - boolean dataExists = totalSize > 0; - if (dataExists) { + // end this batch now. Returning false without data made serialize() wait out + // the async fsync delay (1 s by default) for entries that can no longer come, on every + // close. Pending data or checkpoints are synced here instead of after that wait. + if (totalSize > 0 || !info.checkpoints.isEmpty()) { fsyncWorkingBuffer(currentSearchIndex, currentFileStatus, info); } isClosed = true; - return dataExists; + return true; default: return false; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java index f37931e97e0..6e186815edf 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java @@ -89,6 +89,9 @@ public abstract class AbstractPeriodicalServiceWithAdvance { public void stopService() { allowSubmitListen = false; + // wake the loop from its interval wait, so that it ends now and its executor can + // terminate, instead of only after the next interval + advanceExecution(); LOGGER.info(ServiceMessages.SERVICE_STOPPED_SUCCESSFULLY, getClass().getSimpleName()); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java index 75acc8d6f00..0edc1461654 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java @@ -118,13 +118,17 @@ public abstract class ThriftService implements IService { // sleep 100ms for waiting the rpc server start. Thread.sleep(100); } - } catch (InterruptedException - | ClassNotFoundException + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new StartupException(this.getID().getName(), e.getMessage()); + } catch (ClassNotFoundException | IllegalAccessException | InstantiationException | NoSuchMethodException | InvocationTargetException e) { - Thread.currentThread().interrupt(); + // only an interruption re-sets the interrupt flag; a failed start (e.g. the port is in use, + // reported as IllegalAccessException) must not leave the calling thread interrupted, or the + // cleanup after it runs on an interrupted thread throw new StartupException(this.getID().getName(), e.getMessage()); }
