This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 30173fae954 [Core] Wake the procedure, WAL, disk-usage and periodical
workers on stop (#19177)
30173fae954 is described below
commit 30173fae9540001b47732ee9c8f21f544281ca21
Author: Christofer Dutz <[email protected]>
AuthorDate: Sun Nov 8 13:52:36 2026 +0100
[Core] Wake the procedure, WAL, disk-usage and periodical workers on stop
(#19177)
---
.../confignode/procedure/ProcedureExecutor.java | 6 +-
.../procedure/TimeoutExecutorThread.java | 18 +++-
.../confignode/service/ConfigNodeShutdownHook.java | 3 +-
.../procedure/ProcedureExecutorStopTest.java | 57 +++++++++++++
.../tableDiskUsageIndex/TableDiskUsageIndex.java | 18 ++++
.../dataregion/wal/buffer/WALBuffer.java | 8 +-
.../AbstractPeriodicalServiceWithAdvance.java | 14 +++-
.../iotdb/commons/service/ThriftService.java | 10 ++-
.../AbstractPeriodicalServiceWithAdvanceTest.java | 95 ++++++++++++++++++++++
9 files changed, 218 insertions(+), 11 deletions(-)
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
index 32797ba0c7a..f9a1a06e488 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
@@ -1028,7 +1028,11 @@ public class ProcedureExecutor<Env> {
while (isRunning() && keepAlive(lastUpdated)) {
Procedure<Env> procedure = workerScheduler.poll(keepAliveTime,
TimeUnit.MILLISECONDS);
if (procedure == null) {
- Thread.sleep(1000);
+ // a stopped scheduler returns null at once; leave right away
instead of
+ // sleeping another second first
+ if (isRunning()) {
+ Thread.sleep(1000);
+ }
continue;
}
boolean executionAcquired = false;
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
index 23cfb6c089f..73fe43d2c26 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
@@ -39,6 +39,9 @@ public class TimeoutExecutorThread<Env> extends
StoppableThread {
private final DelayQueue<ProcedureDelayContainer<Env>> queue = new
DelayQueue<>();
private final Set<Procedure<Env>> registeredInternalProcedures =
ConcurrentHashMap.newKeySet();
+ /** Queued by {@link #sendStopSignal()}; due at once, carries no procedure.
*/
+ private final ProcedureDelayContainer<Env> stopMarker = new
ProcedureDelayContainer<>(null);
+
public TimeoutExecutorThread(
ProcedureExecutor<Env> envProcedureExecutor, ThreadGroup threadGroup,
String name) {
super(threadGroup, name);
@@ -78,7 +81,7 @@ public class TimeoutExecutorThread<Env> extends
StoppableThread {
public void run() {
while (executor.isRunning()) {
ProcedureDelayContainer<Env> delayTask = takeQuietly();
- if (delayTask == null) {
+ if (delayTask == null || delayTask == stopMarker) {
continue;
}
Procedure<Env> procedure = delayTask.getProcedure();
@@ -110,7 +113,15 @@ public class TimeoutExecutorThread<Env> extends
StoppableThread {
}
}
- public void sendStopSignal() {}
+ @Override
+ public void sendStopSignal() {
+ // wake run() from its queue poll (up to DELAY_QUEUE_TIMEOUT seconds), so
that it
+ // sees the stopped executor at once. Before, join() waited until
awaitTermination's eighth
+ // join round interrupted the thread, about 2 s per thread on every stop.
+ if (!queue.contains(stopMarker)) {
+ queue.add(stopMarker);
+ }
+ }
private static class ProcedureDelayContainer<Env> implements Delayed {
@@ -143,6 +154,9 @@ public class TimeoutExecutorThread<Env> extends
StoppableThread {
@Override
public long getDelay(TimeUnit unit) {
+ if (procedure == null) {
+ return 0; // the stop marker
+ }
long delay = procedure.getTimeoutTimestamp() -
System.currentTimeMillis();
return unit.convert(delay, TimeUnit.MILLISECONDS);
}
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
index 2296d54cd02..dfac10a0f87 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
@@ -101,7 +101,8 @@ public class ConfigNodeShutdownHook extends Thread {
// redirect. If no leader is reachable, the new leader will mark this node
Unknown by
// heartbeat timeout instead.
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Stopped);
- if (!reportShutdown(localNode, reportTargets)) {
+ // a single ConfigNode has nobody to report to; that is no failure worth
an ERROR
+ if (!reportTargets.isEmpty() && !reportShutdown(localNode, reportTargets))
{
LOGGER.error(
ConfigNodeMessages.REPORTING_CONFIGNODE_SHUTDOWN_FAILED_THE_CLUSTER_WILL_STILL_TAKE_THE);
}
diff --git
a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ProcedureExecutorStopTest.java
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ProcedureExecutorStopTest.java
new file mode 100644
index 00000000000..1d6d98b1356
--- /dev/null
+++
b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/ProcedureExecutorStopTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+import org.apache.iotdb.confignode.procedure.env.TestProcEnv;
+import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
+
+import org.junit.Test;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertTrue;
+
+public class ProcedureExecutorStopTest {
+
+ /**
+ * An idle executor must stop without waiting out the poll intervals of its
threads: the timeout
+ * and worker-monitor threads used to wait until awaitTermination
interrupted them (about 2 s
+ * each), and every worker slept another second after its scheduler was
stopped.
+ */
+ @Test(timeout = 60_000)
+ public void anIdleExecutorStopsWithoutWaitingOutItsPollIntervals() {
+ TestProcEnv env = new TestProcEnv();
+ IProcedureStore store = new NoopProcedureStore();
+ ProcedureExecutor<TestProcEnv> executor = new ProcedureExecutor<>(env,
store);
+ env.setScheduler(executor.getScheduler());
+ executor.init(4);
+ store.start();
+ executor.startWorkers();
+
+ long start = System.nanoTime();
+ executor.stop();
+ executor.join();
+ long tookMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+
+ store.stop();
+ store.cleanup();
+ assertTrue("stop and join took " + tookMillis + " ms", tookMillis < 1_500);
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
index ea578316511..641dd9a4b10 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
@@ -236,6 +236,11 @@ public class TableDiskUsageIndex {
}
try {
stop = true;
+ // wake the worker from its queue poll (up to 1 s) so that it sees the
stop flag
+ if (!queue.offer(new WakeUpOperation())) {
+ // the queue is full, so the worker is not waiting in poll() and needs
no wake-up; it sees
+ // the stop flag after its current operation
+ }
scheduledExecutorService.shutdown();
scheduledExecutorService.awaitTermination(5, TimeUnit.SECONDS);
writerMap.values().forEach(DataRegionTableSizeIndexWriter::close);
@@ -303,6 +308,19 @@ public class TableDiskUsageIndex {
}
}
+ /** Does nothing; {@link #close()} queues it to wake the worker. */
+ private static class WakeUpOperation extends Operation {
+
+ private WakeUpOperation() {
+ super(null, -1);
+ }
+
+ @Override
+ public void apply(TableDiskUsageIndex tableDiskUsageIndex) {
+ // nothing to do
+ }
+ }
+
protected static class StartReadOperation extends Operation {
protected final DataRegion region;
protected final boolean readTsFileIndex;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
index eba379f1cc5..d1ef522f4bf 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
@@ -443,12 +443,14 @@ public class WALBuffer extends AbstractWALBuffer {
identifier,
walEntries.size());
}
- boolean dataExists = totalSize > 0;
- if (dataExists) {
+ // end this batch now. Returning false without data made serialize()
wait out
+ // the async fsync delay (1 s by default) for entries that can no
longer come, on every
+ // close. Pending data or checkpoints are synced here instead of
after that wait.
+ if (totalSize > 0 || !info.checkpoints.isEmpty()) {
fsyncWorkingBuffer(currentSearchIndex, currentFileStatus, info);
}
isClosed = true;
- return dataExists;
+ return true;
default:
return false;
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java
index f37931e97e0..393ccc40d46 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvance.java
@@ -62,7 +62,9 @@ public abstract class AbstractPeriodicalServiceWithAdvance {
lock.lock();
try {
executeTask();
- if (!skipNextSleep) {
+ // checked under the lock that stopService() signals under, so a stop
that came before
+ // this iteration began (e.g. while it was still queued) is not missed
+ if (!skipNextSleep && allowSubmitListen) {
condition.await(intervalMillis, TimeUnit.MILLISECONDS);
}
skipNextSleep = false;
@@ -89,6 +91,16 @@ public abstract class AbstractPeriodicalServiceWithAdvance {
public void stopService() {
allowSubmitListen = false;
+ // wake the loop from its interval wait, so that it ends now and its
executor can
+ // terminate, instead of only after the next interval. Take the lock
unconditionally (unlike
+ // advanceExecution()): a running iteration either sees the stopped state
before it waits or
+ // is already waiting and gets this signal.
+ lock.lock();
+ try {
+ condition.signalAll();
+ } finally {
+ lock.unlock();
+ }
LOGGER.info(ServiceMessages.SERVICE_STOPPED_SUCCESSFULLY,
getClass().getSimpleName());
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java
index 75acc8d6f00..0edc1461654 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ThriftService.java
@@ -118,13 +118,17 @@ public abstract class ThriftService implements IService {
// sleep 100ms for waiting the rpc server start.
Thread.sleep(100);
}
- } catch (InterruptedException
- | ClassNotFoundException
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new StartupException(this.getID().getName(), e.getMessage());
+ } catch (ClassNotFoundException
| IllegalAccessException
| InstantiationException
| NoSuchMethodException
| InvocationTargetException e) {
- Thread.currentThread().interrupt();
+ // only an interruption re-sets the interrupt flag; a failed start (e.g.
the port is in use,
+ // reported as IllegalAccessException) must not leave the calling thread
interrupted, or the
+ // cleanup after it runs on an interrupted thread
throw new StartupException(this.getID().getName(), e.getMessage());
}
diff --git
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvanceTest.java
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvanceTest.java
new file mode 100644
index 00000000000..82d24609d27
--- /dev/null
+++
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/service/AbstractPeriodicalServiceWithAdvanceTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.service;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class AbstractPeriodicalServiceWithAdvanceTest {
+
+ // far longer than the test may take, so only a wake-up can end the interval
wait
+ private static final long INTERVAL_MS = TimeUnit.MINUTES.toMillis(10);
+ private static final long STOP_TIMEOUT_MS = 5_000;
+
+ private final ExecutorService executor = Executors.newSingleThreadExecutor();
+
+ @After
+ public void tearDown() {
+ executor.shutdownNow();
+ }
+
+ @Test
+ public void stopWakesARunningIteration() throws Exception {
+ final CountDownLatch firstRun = new CountDownLatch(1);
+ final TestService service = new TestService(executor, firstRun);
+
+ service.startService();
+ Assert.assertTrue(firstRun.await(STOP_TIMEOUT_MS, TimeUnit.MILLISECONDS));
+ service.stopService();
+ executor.shutdown();
+
+ Assert.assertTrue(executor.awaitTermination(STOP_TIMEOUT_MS,
TimeUnit.MILLISECONDS));
+ Assert.assertEquals(1, service.runs.get());
+ }
+
+ @Test
+ public void stopBeforeTheQueuedIterationBegins() throws Exception {
+ // hold the executor's only thread, so that the iteration submitted by
startService() stays
+ // queued until after stopService()
+ final CountDownLatch release = new CountDownLatch(1);
+ executor.submit(
+ () -> {
+ release.await();
+ return null;
+ });
+ final TestService service = new TestService(executor, new
CountDownLatch(1));
+
+ service.startService();
+ service.stopService();
+ executor.shutdown();
+ release.countDown();
+
+ Assert.assertTrue(executor.awaitTermination(STOP_TIMEOUT_MS,
TimeUnit.MILLISECONDS));
+ Assert.assertEquals(1, service.runs.get());
+ }
+
+ private static class TestService extends
AbstractPeriodicalServiceWithAdvance {
+ private final AtomicInteger runs = new AtomicInteger();
+ private final CountDownLatch firstRun;
+
+ TestService(final ExecutorService executor, final CountDownLatch firstRun)
{
+ super(executor, INTERVAL_MS);
+ this.firstRun = firstRun;
+ }
+
+ @Override
+ protected void executeTask() {
+ runs.incrementAndGet();
+ firstRun.countDown();
+ }
+ }
+}