This is an automated email from the ASF dual-hosted git repository.
CRZbulabula pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new fc6f68c2132 Fix premature recovery from DiskFull read-only mode
(#19126)
fc6f68c2132 is described below
commit fc6f68c213230d0bb166420f901f3bc8b00663ba
Author: d-wang-commit <[email protected]>
AuthorDate: Thu Nov 5 19:20:36 2026 +0800
Fix premature recovery from DiskFull read-only mode (#19126)
* Fix premature recovery from DiskFull read-only mode
Require affected directories to recover before restoring Running status.
Preserve manual and unrecoverable read-only reasons.
* Register disk recovery at the directory selection failure
Let FolderManager check the same directories on failure and recovery.
Remove the outer DataRegion registration and keep the availability check
private.
* Check current tier directories without retrying during DiskFull recovery
---
.../impl/DataNodeInternalRPCServiceImpl.java | 8 +-
.../db/storageengine/dataregion/DataRegion.java | 4 -
.../tsfile/generator/TsFileNameGenerator.java | 2 +-
.../db/storageengine/rescon/disk/TierManager.java | 196 +++++++++++++--------
.../DataNodeInternalRPCServiceImplDiskTest.java | 46 +++++
.../disk/FolderManagerDiskFullRecoveryTest.java | 160 +++++++++++++++++
.../apache/iotdb/commons/conf/CommonConfig.java | 35 ++++
.../apache/iotdb/commons/disk/FolderManager.java | 37 ++--
.../commons/disk/strategy/DirectoryStrategy.java | 22 ++-
.../apache/iotdb/commons/utils/JVMCommonUtils.java | 26 ++-
.../iotdb/commons/conf/CommonConfigTest.java | 130 ++++++++++++++
.../iotdb/commons/utils/JVMCommonUtilsTest.java | 35 ++++
12 files changed, 600 insertions(+), 101 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index b0ec681b09b..836f8d47f29 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -2651,13 +2651,7 @@ public class DataNodeInternalRPCServiceImpl implements
IDataNodeRPCService.Iface
commonConfig.getDiskSpaceWarningThreshold());
commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly,
NodeStatus.DISK_FULL);
} else {
- synchronized (commonConfig) {
- // A concurrent Manual/Stopping/error transition must not be cleared
by disk recovery.
- if (NodeStatus.ReadOnly.equals(commonConfig.getNodeStatus())
- && NodeStatus.DISK_FULL.equals(commonConfig.getStatusReason())) {
- commonConfig.setNodeStatus(NodeStatus.Running);
- }
- }
+ commonConfig.tryRecoverFromDiskFull();
}
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index cdf06c8e4ca..ee877628cd9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -25,7 +25,6 @@ import org.apache.iotdb.calc.utils.ObjectTypeUtils;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.commons.client.exception.ClientManagerException;
-import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
@@ -2239,9 +2238,6 @@ public class DataRegion implements IDataRegionForQuery {
StorageEngineMessages
.STORAGE_LOG_DISK_SPACE_IS_INSUFFICIENT_WHEN_CREATING_TSFILE_PROCESSOR_4032BAF0,
e);
- CommonDescriptor.getInstance()
- .getConfig()
- .setNodeStatusWithReason(NodeStatus.ReadOnly,
NodeStatus.DISK_FULL);
throw new WriteProcessException(e.getMessage(), e.getErrorCode(),
true);
} catch (IOException e) {
if (retryCnt < 3) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
index 29c27ee7978..b64f3436c7f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
@@ -96,8 +96,8 @@ public class TsFileNameGenerator {
int tierLevel,
String customSuffix)
throws DiskSpaceInsufficientException, IOException {
- FolderManager folderManager =
TierManager.getInstance().getFolderManager(tierLevel, sequence);
try {
+ FolderManager folderManager =
TierManager.getInstance().getFolderManager(tierLevel, sequence);
return folderManager.getNextWithRetry(
baseDir -> {
String tsFileDir =
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
index 59bc5877979..b8f3134817c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.disk.FolderManager;
import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType;
import org.apache.iotdb.commons.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.commons.utils.JVMCommonUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
@@ -58,20 +59,12 @@ import java.util.stream.Collectors;
public class TierManager {
private static final Logger logger =
LoggerFactory.getLogger(TierManager.class);
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
- private DirectoryStrategyType directoryStrategyType =
DirectoryStrategyType.SEQUENCE_STRATEGY;
- /**
- * seq folder manager of each storage tier, managing both data directories
and multi-dir strategy
- */
- private volatile List<FolderManager> seqTiers = new ArrayList<>();
+ private record TierDirectory(int tierLevel, String folderName) {}
- /**
- * unSeq folder manager of each storage tier, managing both data directories
and multi-dir
- * strategy
- */
- private volatile List<FolderManager> unSeqTiers = new ArrayList<>();
-
- private volatile List<FolderManager> objectTiers = new ArrayList<>();
+ private volatile Map<TierDirectory, TierFolderManager> folderManagers =
Map.of();
+ // A failed constructor can report DiskFull before the replacement managers
are published.
+ private volatile boolean rebuildingFolders;
/** seq file folder's rawFsPath path -> tier level */
private volatile Map<String, Integer> seqDir2TierLevel = new HashMap<>();
@@ -81,31 +74,27 @@ public class TierManager {
private List<String> objectDirs;
- private List<String> copyToTargetDirs;
-
- // private FolderManager copyToFolderManager;
-
- private MemoizedCheckedSupplier<FolderManager,
DiskSpaceInsufficientException>
- copyToFolderManager;
-
/** total space of each tier, Long.MAX_VALUE when one tier contains remote
storage */
- private long[] tierDiskTotalSpace;
+ private volatile long[] tierDiskTotalSpace = new long[0];
private TierManager() {
initFolders();
}
public synchronized void initFolders() {
- initFolders(seqTiers, unSeqTiers, objectTiers, seqDir2TierLevel,
unSeqDir2TierLevel);
+ rebuildingFolders = true;
+ try {
+ rebuildFolders();
+ } finally {
+ rebuildingFolders = false;
+ }
}
- private void initFolders(
- List<FolderManager> seqTiers,
- List<FolderManager> unSeqTiers,
- List<FolderManager> objectTiers,
- Map<String, Integer> seqDir2TierLevel,
- Map<String, Integer> unSeqDir2TierLevel) {
- directoryStrategyType =
+ private void rebuildFolders() {
+ Map<TierDirectory, TierFolderManager> newManagers = new HashMap<>();
+ Map<String, Integer> newSeqDir2TierLevel = new HashMap<>();
+ Map<String, Integer> newUnSeqDir2TierLevel = new HashMap<>();
+ DirectoryStrategyType directoryStrategyType =
DirectoryStrategyType.fromClassName(config.getMultiDirStrategyClassName());
config.updatePath();
@@ -140,13 +129,14 @@ public class TierManager {
.getPath())
.collect(Collectors.toList());
mkDataDirs(seqDirs);
- try {
- seqTiers.add(new FolderManager(seqDirs, directoryStrategyType));
- } catch (DiskSpaceInsufficientException e) {
- logger.error(StorageEngineMessages.ALL_DISKS_OF_TIER_FULL, tierLevel,
e);
- }
+ addFolderManager(
+ newManagers,
+ tierLevel,
+ IoTDBConstant.SEQUENCE_FOLDER_NAME,
+ seqDirs,
+ directoryStrategyType);
for (String dir : seqDirs) {
- seqDir2TierLevel.put(dir, tierLevel);
+ newSeqDir2TierLevel.put(dir, tierLevel);
}
List<String> unSeqDirs =
@@ -159,17 +149,18 @@ public class TierManager {
.getPath())
.collect(Collectors.toList());
mkDataDirs(unSeqDirs);
- try {
- unSeqTiers.add(new FolderManager(unSeqDirs, directoryStrategyType));
- } catch (DiskSpaceInsufficientException e) {
- logger.error(StorageEngineMessages.ALL_DISKS_OF_TIER_FULL, tierLevel,
e);
- }
+ addFolderManager(
+ newManagers,
+ tierLevel,
+ IoTDBConstant.UNSEQUENCE_FOLDER_NAME,
+ unSeqDirs,
+ directoryStrategyType);
for (String dir : unSeqDirs) {
- unSeqDir2TierLevel.put(dir, tierLevel);
+ newUnSeqDir2TierLevel.put(dir, tierLevel);
}
if (tierLevel == 0) {
- copyToTargetDirs =
+ List<String> copyToTargetDirs =
Arrays.stream(tierDirs[tierLevel])
.filter(Objects::nonNull)
.map(
@@ -178,9 +169,12 @@ public class TierManager {
.getFile(v,
IoTDBConstant.COPY_TO_TARGET_FOLDER_NAME)
.getPath())
.collect(Collectors.toList());
- copyToFolderManager =
- MemoizedCheckedSupplier.valueOf(
- () -> new FolderManager(copyToTargetDirs,
directoryStrategyType));
+ addFolderManager(
+ newManagers,
+ tierLevel,
+ IoTDBConstant.COPY_TO_TARGET_FOLDER_NAME,
+ copyToTargetDirs,
+ directoryStrategyType);
}
objectDirs =
@@ -193,11 +187,12 @@ public class TierManager {
.getPath())
.collect(Collectors.toList());
- try {
- objectTiers.add(new FolderManager(objectDirs, directoryStrategyType));
- } catch (DiskSpaceInsufficientException e) {
- logger.error(StorageEngineMessages.ALL_DISKS_OF_TIER_FULL, tierLevel,
e);
- }
+ addFolderManager(
+ newManagers,
+ tierLevel,
+ IoTDBConstant.OBJECT_FOLDER_NAME,
+ objectDirs,
+ directoryStrategyType);
// try to remove empty objectDirs
for (String dir : objectDirs) {
File dirFile = FSFactoryProducer.getFSFactory().getFile(dir);
@@ -211,23 +206,74 @@ public class TierManager {
}
tierDiskTotalSpace = getTierDiskSpace(DiskSpaceType.TOTAL);
+ seqDir2TierLevel = newSeqDir2TierLevel;
+ unSeqDir2TierLevel = newUnSeqDir2TierLevel;
+ folderManagers = newManagers;
+ }
+
+ private void addFolderManager(
+ Map<TierDirectory, TierFolderManager> newManagers,
+ int tierLevel,
+ String folderName,
+ List<String> folders,
+ DirectoryStrategyType type) {
+ TierDirectory key = new TierDirectory(tierLevel, folderName);
+ TierFolderManager entry = new TierFolderManager(key, folders, type);
+ newManagers.put(key, entry);
+ if (!IoTDBConstant.COPY_TO_TARGET_FOLDER_NAME.equals(folderName)) {
+ try {
+ entry.supplier.get();
+ } catch (DiskSpaceInsufficientException e) {
+ logger.error(StorageEngineMessages.ALL_DISKS_OF_TIER_FULL, tierLevel,
e);
+ }
+ }
+ }
+
+ private final class TierFolderManager {
+ private final List<String> folders;
+ private final MemoizedCheckedSupplier<FolderManager,
DiskSpaceInsufficientException> supplier;
+ private volatile FolderManager manager;
+
+ private TierFolderManager(TierDirectory key, List<String> folders,
DirectoryStrategyType type) {
+ this.folders = folders;
+ supplier =
+ MemoizedCheckedSupplier.valueOf(
+ () -> {
+ manager = new FolderManager(folders, type, () ->
reportDiskFull(key, this));
+ return manager;
+ });
+ }
+
+ private boolean hasSpaceForRecovery() {
+ FolderManager current = manager;
+ return current == null
+ ? folders.stream().anyMatch(JVMCommonUtils::hasSpaceWithoutRetry)
+ : current.hasSpaceForRecovery();
+ }
+ }
+
+ private void reportDiskFull(TierDirectory key, TierFolderManager source) {
+ if (!rebuildingFolders && folderManagers.get(key) != source) {
+ return;
+ }
+ CommonDescriptor.getInstance()
+ .getConfig()
+ .setReadOnlyForDiskFull(key, () -> hasSpaceForRecovery(key));
+ }
+
+ private boolean hasSpaceForRecovery(TierDirectory key) {
+ Map<TierDirectory, TierFolderManager> active = folderManagers;
+ if (rebuildingFolders) {
+ return false;
+ }
+ TierFolderManager current = active.get(key);
+ boolean recovered = current == null || current.hasSpaceForRecovery();
+ return recovered && active == folderManagers && !rebuildingFolders;
}
public synchronized void resetFolders() {
long startTime = System.currentTimeMillis();
- List<FolderManager> newSeqTiers = new ArrayList<>();
- List<FolderManager> newUnSeqTiers = new ArrayList<>();
- List<FolderManager> newObjectTiers = new ArrayList<>();
- Map<String, Integer> newSeqDir2TierLevel = new HashMap<>();
- Map<String, Integer> newUnSeqDir2TierLevel = new HashMap<>();
-
- initFolders(
- newSeqTiers, newUnSeqTiers, newObjectTiers, newSeqDir2TierLevel,
newUnSeqDir2TierLevel);
- seqTiers = newSeqTiers;
- unSeqTiers = newUnSeqTiers;
- objectTiers = newObjectTiers;
- seqDir2TierLevel = newSeqDir2TierLevel;
- unSeqDir2TierLevel = newUnSeqDir2TierLevel;
+ initFolders();
long endTime = System.currentTimeMillis();
logger.info(StorageEngineMessages.FOLDERS_RESET_SUCCESSFULLY, (endTime -
startTime));
}
@@ -251,21 +297,31 @@ public class TierManager {
public String getNextFolderForTsFile(int tierLevel, boolean sequence)
throws DiskSpaceInsufficientException {
- return sequence
- ? seqTiers.get(tierLevel).getNextFolder()
- : unSeqTiers.get(tierLevel).getNextFolder();
+ return getFolderManager(tierLevel, sequence).getNextFolder();
}
public String getNextFolderForCopyToTargetFile() throws
DiskSpaceInsufficientException {
- return copyToFolderManager.get().getNextFolder();
+ return getFolderManager(0,
IoTDBConstant.COPY_TO_TARGET_FOLDER_NAME).getNextFolder();
}
public String getNextFolderForObjectFile() throws
DiskSpaceInsufficientException {
- return objectTiers.get(0).getNextFolder();
+ return getFolderManager(0,
IoTDBConstant.OBJECT_FOLDER_NAME).getNextFolder();
+ }
+
+ public FolderManager getFolderManager(int tierLevel, boolean sequence)
+ throws DiskSpaceInsufficientException {
+ String folderName =
+ sequence ? IoTDBConstant.SEQUENCE_FOLDER_NAME :
IoTDBConstant.UNSEQUENCE_FOLDER_NAME;
+ return getFolderManager(tierLevel, folderName);
}
- public FolderManager getFolderManager(int tierLevel, boolean sequence) {
- return sequence ? seqTiers.get(tierLevel) : unSeqTiers.get(tierLevel);
+ private FolderManager getFolderManager(int tierLevel, String folderName)
+ throws DiskSpaceInsufficientException {
+ TierFolderManager entry = folderManagers.get(new TierDirectory(tierLevel,
folderName));
+ if (entry == null) {
+ throw new IndexOutOfBoundsException(tierLevel);
+ }
+ return entry.supplier.get();
}
public List<String> getAllFilesFolders() {
@@ -356,7 +412,7 @@ public class TierManager {
}
public int getTiersNum() {
- return seqTiers.size();
+ return tierDiskTotalSpace.length;
}
public int getFileTierLevel(File file) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java
index e397baabb16..a6106e0a7b0 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java
@@ -33,6 +33,8 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.util.concurrent.atomic.AtomicBoolean;
+
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -88,6 +90,50 @@ public class DataNodeInternalRPCServiceImplDiskTest {
Assert.assertEquals(0.48, loadSample.getDiskUsageRate(), 1e-10);
}
+ @Test
+ public void testHealthyAggregateStillRequiresEveryFailedSourceToRecover() {
+ AtomicBoolean sequenceAvailable = new AtomicBoolean(false);
+ AtomicBoolean snapshotAvailable = new AtomicBoolean(false);
+ commonConfig.setReadOnlyForDiskFull("sequence", sequenceAvailable::get);
+ commonConfig.setReadOnlyForDiskFull("snapshot", snapshotAvailable::get);
+ SystemMetrics systemMetrics = mock(SystemMetrics.class);
+ when(systemMetrics.getSystemDiskAvailableSpace()).thenReturn(104L);
+ when(systemMetrics.getSystemDiskTotalSpace()).thenReturn(200L);
+ DataNodeInternalRPCServiceImpl service =
+ new DataNodeInternalRPCServiceImpl(mock(DataNodeContext.class),
systemMetrics);
+
+ service.sampleDiskLoad(new TLoadSample());
+ Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL, commonConfig.getStatusReason());
+ snapshotAvailable.set(true);
+ service.sampleDiskLoad(new TLoadSample());
+ Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL, commonConfig.getStatusReason());
+ sequenceAvailable.set(true);
+ service.sampleDiskLoad(new TLoadSample());
+ Assert.assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ Assert.assertNull(commonConfig.getStatusReason());
+ }
+
+ @Test
+ public void testRecoveredSourcesStillRequireHealthyAggregate() {
+ commonConfig.setReadOnlyForDiskFull("sequence", () -> true);
+ SystemMetrics systemMetrics = mock(SystemMetrics.class);
+ when(systemMetrics.getSystemDiskTotalSpace()).thenReturn(100L);
+ DataNodeInternalRPCServiceImpl service =
+ new DataNodeInternalRPCServiceImpl(mock(DataNodeContext.class),
systemMetrics);
+ for (long available : new long[] {0, 4}) {
+ when(systemMetrics.getSystemDiskAvailableSpace()).thenReturn(available);
+ service.sampleDiskLoad(new TLoadSample());
+ Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL,
commonConfig.getStatusReason());
+ }
+ when(systemMetrics.getSystemDiskAvailableSpace()).thenReturn(10L);
+ service.sampleDiskLoad(new TLoadSample());
+ Assert.assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ Assert.assertNull(commonConfig.getStatusReason());
+ }
+
@Test
public void testStorageEngineDiskAggregateStillEntersReadOnly() {
SystemMetrics systemMetrics = mock(SystemMetrics.class);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManagerDiskFullRecoveryTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManagerDiskFullRecoveryTest.java
new file mode 100644
index 00000000000..8c0204230cd
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManagerDiskFullRecoveryTest.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.rescon.disk;
+
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.disk.FolderManager;
+import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType;
+import org.apache.iotdb.commons.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.commons.utils.JVMCommonUtils;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.reflect.Whitebox;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*",
"javax.management.*"})
+@PrepareForTest({CommonDescriptor.class, JVMCommonUtils.class})
+public class FolderManagerDiskFullRecoveryTest {
+ private CommonConfig commonConfig;
+
+ @Before
+ public void setUp() throws Exception {
+ commonConfig = Whitebox.invokeConstructor(CommonConfig.class);
+ PowerMockito.mockStatic(CommonDescriptor.class);
+ CommonDescriptor descriptor = mock(CommonDescriptor.class);
+ when(CommonDescriptor.getInstance()).thenReturn(descriptor);
+ when(descriptor.getConfig()).thenReturn(commonConfig);
+ PowerMockito.mockStatic(JVMCommonUtils.class);
+ }
+
+ @Test
+ public void testFullSnapshotDirectoryDoesNotRecoverWhileFull() throws
Exception {
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ List<String> snapshots =
+ Arrays.asList("bounded-data/snapshot", "second-disk/snapshot",
"third-disk/snapshot");
+ for (int i = 0; i < 3; i++) {
+ // Constructor failure must be remembered even though no FolderManager
is returned.
+ Assert.assertThrows(
+ DiskSpaceInsufficientException.class,
+ () -> new FolderManager(snapshots,
DirectoryStrategyType.SEQUENCE_STRATEGY));
+ commonConfig.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL,
commonConfig.getStatusReason());
+ }
+
+ // Each failed construction uses the retrying write-path probe. Each
recovery checks once.
+ for (String snapshot : snapshots) {
+ PowerMockito.verifyStatic(JVMCommonUtils.class, times(3));
+ JVMCommonUtils.hasSpace(snapshot);
+ PowerMockito.verifyStatic(JVMCommonUtils.class, times(3));
+ JVMCommonUtils.hasSpaceWithoutRetry(snapshot);
+ }
+
when(JVMCommonUtils.hasSpaceWithoutRetry(snapshots.get(0))).thenReturn(true);
+ commonConfig.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ Assert.assertNull(commonConfig.getStatusReason());
+
+ // A later business operation must still trigger protection if the
directory becomes full again.
+
when(JVMCommonUtils.hasSpaceWithoutRetry(snapshots.get(0))).thenReturn(false);
+ Assert.assertThrows(
+ DiskSpaceInsufficientException.class,
+ () -> new FolderManager(snapshots,
DirectoryStrategyType.SEQUENCE_STRATEGY));
+ commonConfig.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL, commonConfig.getStatusReason());
+ }
+
+ @Test
+ public void testAllFailedGroupsMustRecoverButOneCandidatePerGroupIsEnough()
throws Exception {
+ for (String[] group : new String[][] {{"tier0/sequence",
"tier0-spare/sequence"}, {"wal"}}) {
+ Assert.assertThrows(
+ DiskSpaceInsufficientException.class,
+ () -> new FolderManager(Arrays.asList(group),
DirectoryStrategyType.SEQUENCE_STRATEGY));
+ }
+
when(JVMCommonUtils.hasSpaceWithoutRetry("tier0-spare/sequence")).thenReturn(true);
+ commonConfig.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL, commonConfig.getStatusReason());
+
+ when(JVMCommonUtils.hasSpaceWithoutRetry("wal")).thenReturn(true);
+ commonConfig.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ Assert.assertNull(commonConfig.getStatusReason());
+ }
+
+ @Test
+ public void testRuntimeFailureCannotRecoverThroughAnAbnormalFolder() throws
Exception {
+ String full = "sequence/full";
+ String abnormal = "sequence/abnormal";
+ when(JVMCommonUtils.hasSpace(full)).thenReturn(true);
+ when(JVMCommonUtils.hasSpace(abnormal)).thenReturn(true);
+ FolderManager manager =
+ new FolderManager(Arrays.asList(full, abnormal),
DirectoryStrategyType.SEQUENCE_STRATEGY);
+ manager.updateFolderState(abnormal, FolderManager.FolderState.ABNORMAL);
+ when(JVMCommonUtils.hasSpace(full)).thenReturn(false);
+ Assert.assertThrows(DiskSpaceInsufficientException.class,
manager::getNextFolder);
+ when(JVMCommonUtils.hasSpaceWithoutRetry(abnormal)).thenReturn(true);
+
+ commonConfig.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL, commonConfig.getStatusReason());
+
+ when(JVMCommonUtils.hasSpaceWithoutRetry(full)).thenReturn(true);
+ commonConfig.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ Assert.assertNull(commonConfig.getStatusReason());
+ }
+
+ @Test
+ public void testOptOutDirectoryFailureDoesNotBlockRecovery() throws
Exception {
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ Assert.assertThrows(
+ DiskSpaceInsufficientException.class,
+ () ->
+ new FolderManager(
+ Collections.singletonList("load-staging"),
+ DirectoryStrategyType.SEQUENCE_STRATEGY,
+ false));
+ Assert.assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ Assert.assertNull(commonConfig.getStatusReason());
+
+ commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly,
NodeStatus.DISK_FULL);
+ commonConfig.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ Assert.assertNull(commonConfig.getStatusReason());
+ }
+}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index bd8b2e2b6c9..e9dddb4a853 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -41,10 +41,13 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
import java.util.regex.Pattern;
import static org.apache.iotdb.commons.conf.IoTDBConstant.KB;
@@ -185,6 +188,10 @@ public class CommonConfig {
private volatile String statusReason = null;
+ // Only failed sources are retained. Directory initialization uses its
folder list as the key;
+ // a live FolderManager uses its identity so its health restrictions are
also checked.
+ private final Map<Object, BooleanSupplier> diskFullRecoveryChecks = new
HashMap<>();
+
private final int TTimePartitionSlotTransmitLimit = 1000;
/** Disk Monitor. */
@@ -861,6 +868,33 @@ public class CommonConfig {
setNodeStatusWithReason(newStatus, null);
}
+ /** Records the recovery condition together with the disk-full transition. */
+ public synchronized void setReadOnlyForDiskFull(Object source,
BooleanSupplier recoveryCheck) {
+ diskFullRecoveryChecks.put(source, recoveryCheck);
+ setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL);
+ }
+
+ /** Called after the aggregate disk-space check passes. */
+ public void tryRecoverFromDiskFull() {
+ final List<BooleanSupplier> recoveryChecks;
+ synchronized (this) {
+ if (status != NodeStatus.ReadOnly ||
!NodeStatus.DISK_FULL.equals(statusReason)) {
+ return;
+ }
+ recoveryChecks = new ArrayList<>(diskFullRecoveryChecks.values());
+ }
+ // Do not hold the status monitor during filesystem I/O.
+ if (!recoveryChecks.stream().allMatch(BooleanSupplier::getAsBoolean)) {
+ return;
+ }
+ synchronized (this) {
+ // A concurrent Manual/Stopping/error transition must not be cleared by
disk recovery.
+ if (status == NodeStatus.ReadOnly &&
NodeStatus.DISK_FULL.equals(statusReason)) {
+ setNodeStatus(NodeStatus.Running);
+ }
+ }
+ }
+
/**
* Sets the node status and reason together. ReadOnly-to-ReadOnly updates
select the reason
* through {@link NodeStatus#transitionReadOnlyReason(String, String)}.
Other statuses clear the
@@ -868,6 +902,7 @@ public class CommonConfig {
*/
public synchronized void setNodeStatusWithReason(NodeStatus newStatus,
String newReason) {
if (newStatus != NodeStatus.ReadOnly) {
+ diskFullRecoveryChecks.clear();
newReason = null;
} else if (status == NodeStatus.ReadOnly) {
newReason = NodeStatus.transitionReadOnlyReason(statusReason, newReason);
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
index ecb25de8897..4dee48cc47c 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.commons.disk;
-import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.disk.strategy.DirectoryStrategy;
import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType;
import org.apache.iotdb.commons.disk.strategy.MaxDiskUsableSpaceFirstStrategy;
@@ -40,9 +38,10 @@ import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Predicate;
public class FolderManager {
private static final Logger logger =
LoggerFactory.getLogger(FolderManager.class);
@@ -61,7 +60,7 @@ public class FolderManager {
* Map storing the state of each folder (HEALTHY/ABNORMAL). Key: folder path
as String Value:
* corresponding FolderState enum value
*/
- private final Map<String, FolderState> foldersStates = new HashMap<>();
+ private final Map<String, FolderState> foldersStates = new
ConcurrentHashMap<>();
private final DirectoryStrategy selectStrategy;
@@ -75,6 +74,21 @@ public class FolderManager {
public FolderManager(
List<String> folders, DirectoryStrategyType type, boolean
changeSystemStatusToReadOnly)
throws DiskSpaceInsufficientException {
+ this(folders, type, changeSystemStatusToReadOnly, null);
+ }
+
+ /** For an owner that replaces this manager on configuration reload. */
+ public FolderManager(List<String> folders, DirectoryStrategyType type,
Runnable diskFullHandler)
+ throws DiskSpaceInsufficientException {
+ this(folders, type, true, diskFullHandler);
+ }
+
+ private FolderManager(
+ List<String> folders,
+ DirectoryStrategyType type,
+ boolean changeSystemStatusToReadOnly,
+ Runnable diskFullHandler)
+ throws DiskSpaceInsufficientException {
this.folders = folders;
this.changeSystemStatusToReadOnly = changeSystemStatusToReadOnly;
folders.forEach(dir -> foldersStates.put(dir, FolderState.HEALTHY));
@@ -95,6 +109,7 @@ public class FolderManager {
throw new RuntimeException();
}
this.selectStrategy.setChangeSystemStatusToReadOnly(changeSystemStatusToReadOnly);
+ this.selectStrategy.setDiskFullHandler(diskFullHandler);
this.selectStrategy.setFolders(folders);
this.selectStrategy.setFoldersStates(foldersStates);
}
@@ -120,12 +135,16 @@ public class FolderManager {
foldersStates.getOrDefault(folder, FolderState.ABNORMAL) ==
FolderState.HEALTHY);
}
- private boolean hasFolderWithAvailableDiskSpace() {
+ private boolean hasFolderWithAvailableDiskSpace(Predicate<String> hasSpace) {
return folders.stream()
.anyMatch(
folder ->
foldersStates.getOrDefault(folder, FolderState.ABNORMAL) ==
FolderState.HEALTHY
- && JVMCommonUtils.hasSpace(folder));
+ && hasSpace.test(folder));
+ }
+
+ public boolean hasSpaceForRecovery() {
+ return
hasFolderWithAvailableDiskSpace(JVMCommonUtils::hasSpaceWithoutRetry);
}
private void changeToReadOnlyIfDiskFull(DiskSpaceInsufficientException e) {
@@ -133,13 +152,11 @@ public class FolderManager {
return;
}
- if (!hasFolderWithAvailableDiskSpace()) {
+ if (!hasFolderWithAvailableDiskSpace(JVMCommonUtils::hasSpace)) {
if
(LoggerPeriodicalLogReducer.shouldLog(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY))
{
logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e);
}
- CommonDescriptor.getInstance()
- .getConfig()
- .setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL);
+ selectStrategy.reportDiskFull(this, this::hasSpaceForRecovery);
} else {
logger.warn(UtilMessages.CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE, e);
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java
index 978e7ecbbad..4e1072d07d4 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.commons.disk.strategy;
-import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.disk.FolderManager;
import org.apache.iotdb.commons.exception.DiskSpaceInsufficientException;
@@ -33,6 +32,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.BooleanSupplier;
import static org.apache.iotdb.commons.disk.FolderManager.FolderState.HEALTHY;
@@ -49,6 +49,20 @@ public abstract class DirectoryStrategy {
private boolean changeSystemStatusToReadOnly = true;
+ private Runnable diskFullHandler;
+
+ public void setDiskFullHandler(Runnable diskFullHandler) {
+ this.diskFullHandler = diskFullHandler;
+ }
+
+ public void reportDiskFull(Object source, BooleanSupplier recoveryCheck) {
+ if (diskFullHandler != null) {
+ diskFullHandler.run();
+ } else {
+
CommonDescriptor.getInstance().getConfig().setReadOnlyForDiskFull(source,
recoveryCheck);
+ }
+ }
+
public void setChangeSystemStatusToReadOnly(boolean
changeSystemStatusToReadOnly) {
this.changeSystemStatusToReadOnly = changeSystemStatusToReadOnly;
}
@@ -70,9 +84,9 @@ public abstract class DirectoryStrategy {
if (!hasSpace) {
if (changeSystemStatusToReadOnly) {
LOGGER.error(UtilMessages.DISK_SPACE_INSUFFICIENT_READ_ONLY);
- CommonDescriptor.getInstance()
- .getConfig()
- .setNodeStatusWithReason(NodeStatus.ReadOnly,
NodeStatus.DISK_FULL);
+ final List<String> fullFolders = List.copyOf(folders);
+ reportDiskFull(
+ fullFolders, () ->
fullFolders.stream().anyMatch(JVMCommonUtils::hasSpaceWithoutRetry));
} else {
if (LoggerPeriodicalLogReducer.shouldLog(
UtilMessages.MESSAGE_DISK_SPACE_INSUFFICIENT_DF6205B0)) {
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
index 8ef3fe561c7..44135675d61 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
@@ -81,11 +81,7 @@ public class JVMCommonUtils {
public static double getDiskFreeRatio(String dir) {
try {
- File dirFile = FSFactoryProducer.getFSFactory().getFile(dir);
- if (!dirFile.mkdirs()) {
- // This may solve getFreeSpace() == 0?
- dirFile = new File(dir);
- }
+ File dirFile = getDiskSpaceFile(dir);
long freeSpace =
IOUtils.retryNoException(5, 2000L, dirFile::getFreeSpace, space ->
space > 0).orElse(0L);
if (freeSpace == 0) {
@@ -107,6 +103,26 @@ public class JVMCommonUtils {
return getDiskFreeRatio(dir) > diskSpaceWarningThreshold;
}
+ /** Probe once during heartbeat recovery. A later heartbeat can retry a
failed probe. */
+ public static boolean hasSpaceWithoutRetry(String dir) {
+ try {
+ File dirFile = getDiskSpaceFile(dir);
+ long freeSpace = dirFile.getFreeSpace();
+ long totalSpace = dirFile.getTotalSpace();
+ return freeSpace > 0
+ && totalSpace > 0
+ && (double) freeSpace / totalSpace > diskSpaceWarningThreshold;
+ } catch (Exception e) {
+ LOGGER.error(UtilMessages.UNEXPECTED_ERROR_CHECKING_DISK_SPACE, dir, e);
+ return false;
+ }
+ }
+
+ private static File getDiskSpaceFile(String dir) {
+ File dirFile = FSFactoryProducer.getFSFactory().getFile(dir);
+ return dirFile.mkdirs() ? dirFile : new File(dir);
+ }
+
public static long getOccupiedSpace(String folderPath) throws IOException {
Path folder = Paths.get(folderPath);
if (!Files.exists(folder)) {
diff --git
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java
index 86d959c6d55..68a3a98cf49 100644
---
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java
+++
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java
@@ -25,6 +25,8 @@ import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -156,6 +158,134 @@ public class CommonConfigTest {
"UnrecoverableError, 2026-09-02 10:00:00.000, broken",
config.getStatusReason());
}
+ @Test
+ public void testDiskFullRecoveryRequiresEverySourceToRecover() {
+ CommonConfig config = new CommonConfig();
+ AtomicBoolean dataAvailable = new AtomicBoolean(true);
+ AtomicBoolean snapshotAvailable = new AtomicBoolean(false);
+ AtomicInteger dataChecks = new AtomicInteger();
+ AtomicInteger snapshotChecks = new AtomicInteger();
+ config.setReadOnlyForDiskFull(
+ "data",
+ () -> {
+ dataChecks.incrementAndGet();
+ return dataAvailable.get();
+ });
+ config.setReadOnlyForDiskFull(
+ "snapshot",
+ () -> {
+ snapshotChecks.incrementAndGet();
+ return snapshotAvailable.get();
+ });
+
+ config.tryRecoverFromDiskFull();
+ Assert.assertEquals(1, snapshotChecks.get());
+ Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL, config.getStatusReason());
+
+ dataAvailable.set(false);
+ snapshotAvailable.set(true);
+ int previousDataChecks = dataChecks.get();
+ config.tryRecoverFromDiskFull();
+ Assert.assertEquals(previousDataChecks + 1, dataChecks.get());
+ Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL, config.getStatusReason());
+
+ dataAvailable.set(true);
+ previousDataChecks = dataChecks.get();
+ int previousSnapshotChecks = snapshotChecks.get();
+ config.tryRecoverFromDiskFull();
+ Assert.assertEquals(previousDataChecks + 1, dataChecks.get());
+ Assert.assertEquals(previousSnapshotChecks + 1, snapshotChecks.get());
+ Assert.assertEquals(NodeStatus.Running, config.getNodeStatus());
+ Assert.assertNull(config.getStatusReason());
+ }
+
+ @Test
+ public void testRepeatedDiskFullSourceReplacesItsRecoveryCheck() {
+ CommonConfig config = new CommonConfig();
+ Object source = new Object();
+ AtomicInteger previousChecks = new AtomicInteger();
+ AtomicInteger latestChecks = new AtomicInteger();
+ config.setReadOnlyForDiskFull(
+ source,
+ () -> {
+ previousChecks.incrementAndGet();
+ return false;
+ });
+ config.setReadOnlyForDiskFull(
+ source,
+ () -> {
+ latestChecks.incrementAndGet();
+ return true;
+ });
+
+ config.tryRecoverFromDiskFull();
+
+ Assert.assertEquals(0, previousChecks.get());
+ Assert.assertEquals(1, latestChecks.get());
+ Assert.assertEquals(NodeStatus.Running, config.getNodeStatus());
+ Assert.assertNull(config.getStatusReason());
+ }
+
+ @Test
+ public void testDiskRecoveryCannotClearConcurrentHigherPriorityReason() {
+ for (String reason :
+ new String[] {NodeStatus.MANUAL, NodeStatus.UNRECOVERABLE_ERROR + ",
broken"}) {
+ CommonConfig config = new CommonConfig();
+ AtomicInteger checks = new AtomicInteger();
+ config.setReadOnlyForDiskFull(
+ "data",
+ () -> {
+ checks.incrementAndGet();
+ config.setNodeStatusWithReason(NodeStatus.ReadOnly, reason);
+ return true;
+ });
+
+ config.tryRecoverFromDiskFull();
+ Assert.assertEquals(1, checks.get());
+ Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus());
+ Assert.assertEquals(reason, config.getStatusReason());
+
+ config.tryRecoverFromDiskFull();
+ Assert.assertEquals(1, checks.get());
+ Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus());
+ Assert.assertEquals(reason, config.getStatusReason());
+ }
+ }
+
+ @Test
+ public void testManualRunningClearsPreviousDiskRecoveryChecks() {
+ CommonConfig config = new CommonConfig();
+ AtomicInteger previousChecks = new AtomicInteger();
+ AtomicInteger currentChecks = new AtomicInteger();
+ config.setReadOnlyForDiskFull(
+ "old data",
+ () -> {
+ previousChecks.incrementAndGet();
+ return false;
+ });
+ config.tryRecoverFromDiskFull();
+ Assert.assertEquals(1, previousChecks.get());
+ Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus());
+
+ config.setNodeStatus(NodeStatus.Running);
+ Assert.assertEquals(NodeStatus.Running, config.getNodeStatus());
+ Assert.assertNull(config.getStatusReason());
+ config.setReadOnlyForDiskFull(
+ "new data",
+ () -> {
+ currentChecks.incrementAndGet();
+ return true;
+ });
+ config.tryRecoverFromDiskFull();
+
+ Assert.assertEquals(1, previousChecks.get());
+ Assert.assertEquals(1, currentChecks.get());
+ Assert.assertEquals(NodeStatus.Running, config.getNodeStatus());
+ Assert.assertNull(config.getStatusReason());
+ }
+
@Test
public void testHandleUnrecoverableErrorBuildsStatusReason() {
// HandleSystemErrorStrategy writes the singleton config held by
CommonDescriptor.
diff --git
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java
index e314093d9f4..77ffe775b79 100644
---
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java
+++
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java
@@ -19,6 +19,10 @@
package org.apache.iotdb.commons.utils;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+
+import org.apache.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.tsfile.fileSystem.fsFactory.FSFactory;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -33,6 +37,10 @@ import java.util.Comparator;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Stream;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
public class JVMCommonUtilsTest {
@Rule public TemporaryFolder tempFolder = new TemporaryFolder();
@@ -53,6 +61,33 @@ public class JVMCommonUtilsTest {
}
}
+ @Test
+ public void hasSpaceWithoutRetrySamplesOnceAndRequiresSpaceAboveThreshold() {
+ String path = "data/sequence";
+ File folder = mock(File.class);
+ when(folder.mkdirs()).thenReturn(true);
+ when(folder.getFreeSpace()).thenReturn(0L, 5L, 6L);
+ when(folder.getTotalSpace()).thenReturn(100L);
+ FSFactory factory = mock(FSFactory.class);
+ when(factory.getFile(path)).thenReturn(folder);
+ FSFactory originalFactory = FSFactoryProducer.getFSFactory();
+ double originalThreshold =
+
CommonDescriptor.getInstance().getConfig().getDiskSpaceWarningThreshold();
+ try {
+ JVMCommonUtils.setDiskSpaceWarningThreshold(0.05);
+ FSFactoryProducer.setFsFactory(factory);
+ Assert.assertFalse(JVMCommonUtils.hasSpaceWithoutRetry(path));
+ verify(folder).getFreeSpace();
+ Assert.assertFalse(JVMCommonUtils.hasSpaceWithoutRetry(path));
+ Assert.assertTrue(JVMCommonUtils.hasSpaceWithoutRetry(path));
+ when(folder.getTotalSpace()).thenReturn(0L);
+ Assert.assertFalse(JVMCommonUtils.hasSpaceWithoutRetry(path));
+ } finally {
+ FSFactoryProducer.setFsFactory(originalFactory);
+ JVMCommonUtils.setDiskSpaceWarningThreshold(originalThreshold);
+ }
+ }
+
@Test
public void getOccupiedSpaceMissingFolderReturnsZero() throws IOException {
File missing = new File(tempFolder.getRoot(), "does-not-exist");