This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d10b69ee10f Fix tier folder recovery and DiskFull status after
configuration reload (#19161)
d10b69ee10f is described below
commit d10b69ee10ff393928af2dad9725df41b2c28dfc
Author: Hongzhi Gao <[email protected]>
AuthorDate: Tue Nov 10 09:40:11 2026 +0800
Fix tier folder recovery and DiskFull status after configuration reload
(#19161)
* Preserve tier folder slots after disk-full initialization
* Serialize retries of failed tier managers with folder reset
* Keep DiskFull until current directory checks recover
* Add integration coverage for disk-full folder recovery
* Trigger CI rerun
* Trigger CI rerun
* Refresh reloadable folder recovery checks
---
.../iotdb/db/it/IoTDBDiskFullFolderRecoveryIT.java | 158 ++++++++++++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 -
.../iotdb/db/storageengine/StorageEngine.java | 6 -
.../dataregion/snapshot/SnapshotLoader.java | 17 +-
.../db/storageengine/load/LoadStagingDirs.java | 16 +-
.../iotdb/db/storageengine/load/util/LoadUtil.java | 18 +-
.../db/storageengine/rescon/disk/TierManager.java | 198 +++++++++++++--------
.../iotdb/db/conf/DiskFullHotReloadTest.java | 19 +-
.../snapshot/SnapshotLoaderDiskFullReloadTest.java | 126 +++++++++++++
.../rescon/disk/TierManagerDiskFullReloadTest.java | 120 +++++++++++++
.../storageengine/rescon/disk/TierManagerTest.java | 50 ++++++
.../apache/iotdb/commons/conf/CommonConfig.java | 28 ++-
.../apache/iotdb/commons/disk/FolderManager.java | 26 ++-
.../commons/disk/strategy/DirectoryStrategy.java | 22 ++-
.../iotdb/commons/conf/CommonConfigTest.java | 44 +++++
15 files changed, 738 insertions(+), 114 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDiskFullFolderRecoveryIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDiskFullFolderRecoveryIT.java
new file mode 100644
index 00000000000..437df878f95
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDiskFullFolderRecoveryIT.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+
+import org.awaitility.Awaitility;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+
+import static org.apache.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({LocalStandaloneIT.class})
+public class IoTDBDiskFullFolderRecoveryIT {
+ private static final String DATABASE = "root.disk_full_folder_recovery";
+ private static final String FULL_THRESHOLD = "0.9999999999999999";
+ private static final String NORMAL_THRESHOLD = "0.05";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testNewTsFileAfterFullDirectoryReloadAndRecovery() throws
Exception {
+ int dataNodeId = EnvFactory.getEnv().getConfigNodeWrapperList().size();
+ File dataDirectory = new
File(EnvFactory.getEnv().getDataNodeWrapper(0).getDataPath());
+ Assert.assertTrue(dataDirectory.mkdirs() || dataDirectory.isDirectory());
+ Assert.assertTrue(dataDirectory.getTotalSpace() > 0);
+ Assert.assertTrue(
+ (double) dataDirectory.getFreeSpace() / dataDirectory.getTotalSpace()
+ < Double.parseDouble(FULL_THRESHOLD));
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + DATABASE);
+ statement.execute("CREATE TIMESERIES " + DATABASE + ".d1.s1 WITH
DATATYPE=INT64");
+ try {
+ // The first reload installs the threshold; the second rebuilds
FolderManagers under it.
+ setDataNodeConfiguration(
+ statement, dataNodeId, "disk_space_warning_threshold",
FULL_THRESHOLD);
+ setDataNodeConfiguration(statement, dataNodeId,
"enable_partial_insert", "true");
+ awaitDataNodeStatus(statement, dataNodeId, "ReadOnly", "DiskFull");
+
+ // A successful reload of the same configuration must not clear the
full-disk status.
+ setDataNodeConfiguration(statement, dataNodeId,
"enable_partial_insert", "true");
+ Assert.assertEquals("ReadOnly", getDataNodeStatus(statement,
dataNodeId));
+ } finally {
+ setDataNodeConfiguration(
+ statement, dataNodeId, "disk_space_warning_threshold",
NORMAL_THRESHOLD);
+ }
+
+ awaitDataNodeStatus(statement, dataNodeId, "Running", null);
+ statement.execute("INSERT INTO " + DATABASE + ".d1(time,s1)
VALUES(1,1)");
+ statement.execute("FLUSH");
+
+ try (ResultSet resultSet = statement.executeQuery("SELECT s1 FROM " +
DATABASE + ".d1")) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(1, resultSet.getLong("Time"));
+ Assert.assertEquals(1, resultSet.getLong(DATABASE + ".d1.s1"));
+ Assert.assertFalse(resultSet.next());
+ }
+ Assert.assertTrue(hasSequenceTsFile());
+ }
+ }
+
+ private static void setDataNodeConfiguration(
+ Statement statement, int nodeId, String key, String value) throws
Exception {
+ statement.execute("SET CONFIGURATION \"" + key + "\"=\"" + value + "\" ON
" + nodeId);
+ }
+
+ private static void awaitDataNodeStatus(
+ Statement statement, int nodeId, String expectedStatus, String
expectedReason) {
+ Awaitility.await()
+ .atMost(30, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ try (ResultSet resultSet = statement.executeQuery("SHOW
DATANODES")) {
+ while (resultSet.next()) {
+ if (resultSet.getInt("NodeID") == nodeId) {
+ if (!expectedStatus.equals(resultSet.getString("Status")))
{
+ return false;
+ }
+ return expectedReason == null
+ ||
expectedReason.equals(resultSet.getString("StatusReason"));
+ }
+ }
+ }
+ return false;
+ });
+ }
+
+ private static String getDataNodeStatus(Statement statement, int nodeId)
throws Exception {
+ try (ResultSet resultSet = statement.executeQuery("SHOW DATANODES")) {
+ while (resultSet.next()) {
+ if (resultSet.getInt("NodeID") == nodeId) {
+ return resultSet.getString("Status");
+ }
+ }
+ }
+ Assert.fail("DataNode not found: " + nodeId);
+ return null;
+ }
+
+ private static boolean hasSequenceTsFile() throws Exception {
+ Path sequenceDirectory =
+ Paths.get(
+ EnvFactory.getEnv().getDataNodeWrapper(0).getDataPath(),
+ IoTDBConstant.SEQUENCE_FOLDER_NAME,
+ DATABASE);
+ if (!Files.exists(sequenceDirectory)) {
+ return false;
+ }
+ try (Stream<Path> paths = Files.walk(sequenceDirectory)) {
+ return paths.anyMatch(path -> path.toString().endsWith(TSFILE_SUFFIX));
+ }
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 08db154326b..04fe3381b75 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -2517,7 +2517,6 @@ public class IoTDBDescriptor {
return;
}
- boolean wasReadOnlyForDiskFull = commonConfig.isReadOnlyForDiskFull();
TrimProperties commonProperties = new TrimProperties();
try (InputStream inputStream = url.openStream()) {
LOGGER.info(DataNodeMiscMessages.START_RELOAD_CONFIG_FILE, url);
@@ -2535,9 +2534,6 @@ public class IoTDBDescriptor {
ConfigurationFileUtils.releaseDefault();
}
reloadMetricProperties(commonProperties);
- if (wasReadOnlyForDiskFull) {
- commonConfig.setRunningIfDiskFull();
- }
}
public void reloadMetricProperties(TrimProperties properties) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index 505ac60a26c..6c128c027a4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -815,8 +815,6 @@ public class StorageEngine implements IService {
return tsStatus;
}
- CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig();
- boolean wasReadOnlyForDiskFull = commonConfig.isReadOnlyForDiskFull();
try {
ConfigurationFileUtils.updateConfiguration(
new File(configFileUrl.getFile()),
@@ -834,10 +832,6 @@ public class StorageEngine implements IService {
}
return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR,
e.getMessage());
}
- // A SET can apply in memory and still fail when persisting the
configuration file.
- if (wasReadOnlyForDiskFull) {
- commonConfig.setRunningIfDiskFull();
- }
return tsStatus;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
index f029578cda5..dcf2246036c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
@@ -19,10 +19,12 @@
package org.apache.iotdb.db.storageengine.dataregion.snapshot;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.disk.FolderManager;
import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType;
import org.apache.iotdb.commons.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.commons.utils.JVMCommonUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.storageengine.StorageEngine;
@@ -308,7 +310,7 @@ public class SnapshotLoader {
}
}
- private void createLinksFromSnapshotDirToDataDirWithoutLog(
+ void createLinksFromSnapshotDirToDataDirWithoutLog(
File sourceDir, Map<String, String> fileTarget, boolean
preferKeepSameDiskWhenLoading)
throws IOException, DiskSpaceInsufficientException {
if (!sourceDir.exists()) {
@@ -339,7 +341,8 @@ public class SnapshotLoader {
FolderManager folderManager =
new FolderManager(
Arrays.asList(IoTDBDescriptor.getInstance().getConfig().getLocalDataDirs()),
- DirectoryStrategyType.SEQUENCE_STRATEGY);
+ DirectoryStrategyType.SEQUENCE_STRATEGY,
+ SnapshotLoader::reportDiskFull);
File[] timePartitionFolders = seqFileDir.listFiles();
if (timePartitionFolders != null) {
for (File timePartitionFolder : timePartitionFolders) {
@@ -381,6 +384,16 @@ public class SnapshotLoader {
}
}
+ private static void reportDiskFull() {
+ CommonDescriptor.getInstance()
+ .getConfig()
+ .setReadOnlyForDiskFull(
+ SnapshotLoader.class,
+ () ->
+
Arrays.stream(IoTDBDescriptor.getInstance().getConfig().getLocalDataDirs())
+ .anyMatch(JVMCommonUtils::hasSpaceWithoutRetry));
+ }
+
private File createLinksFromSnapshotToSourceDir(
String targetSuffix,
File file,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadStagingDirs.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadStagingDirs.java
index db7544e63e8..911e36ecc87 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadStagingDirs.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadStagingDirs.java
@@ -19,10 +19,12 @@
package org.apache.iotdb.db.storageengine.load;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.disk.FolderManager;
import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType;
import org.apache.iotdb.commons.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.commons.utils.JVMCommonUtils;
import org.apache.iotdb.commons.utils.RetryUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -175,7 +177,9 @@ final class LoadStagingDirs {
|| currentContext.folderManager() == null) {
final FolderManager newFolderManager =
new FolderManager(
- Arrays.asList(currentConfiguredDirs),
DirectoryStrategyType.SEQUENCE_STRATEGY);
+ Arrays.asList(currentConfiguredDirs),
+ DirectoryStrategyType.SEQUENCE_STRATEGY,
+ LoadStagingDirs::reportDiskFull);
CONTEXT_REF.set(new StagingDirectoryContext(currentConfiguredDirs,
newFolderManager));
return newFolderManager;
}
@@ -184,6 +188,16 @@ final class LoadStagingDirs {
return currentContext.folderManager();
}
+ private static void reportDiskFull() {
+ CommonDescriptor.getInstance()
+ .getConfig()
+ .setReadOnlyForDiskFull(
+ LoadStagingDirs.class,
+ () ->
+ Arrays.stream(CONFIG.getLoadTsFileDirs())
+ .anyMatch(JVMCommonUtils::hasSpaceWithoutRetry));
+ }
+
/**
* Immutable snapshot holding synchronized base directories, parsed
normalized paths, and folder
* manager.
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/util/LoadUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/util/LoadUtil.java
index 1ba1a840e09..f5e08c937e1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/util/LoadUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/util/LoadUtil.java
@@ -19,10 +19,12 @@
package org.apache.iotdb.db.storageengine.load.util;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.disk.FolderManager;
import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType;
import org.apache.iotdb.commons.exception.DiskSpaceInsufficientException;
import org.apache.iotdb.commons.utils.FileUtils;
+import org.apache.iotdb.commons.utils.JVMCommonUtils;
import org.apache.iotdb.commons.utils.RetryUtils;
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -453,7 +455,10 @@ public class LoadUtil {
try {
folderManager =
- new FolderManager(Arrays.asList(dirs),
DirectoryStrategyType.SEQUENCE_STRATEGY);
+ new FolderManager(
+ Arrays.asList(dirs),
+ DirectoryStrategyType.SEQUENCE_STRATEGY,
+ LoadUtil::reportDiskFull);
} catch (DiskSpaceInsufficientException e) {
// It should be noted that if this exception is not ignored, the entire
process may fail to
// start.
@@ -477,4 +482,15 @@ public class LoadUtil {
LoadUtil.loadDiskSelector = loadDiskSelector;
return loadDiskSelector;
}
+
+ private static void reportDiskFull() {
+ CommonDescriptor.getInstance()
+ .getConfig()
+ .setReadOnlyForDiskFull(
+ LoadUtil.class,
+ () ->
+ Arrays.stream(
+
IoTDBDescriptor.getInstance().getConfig().getLoadActiveListeningDirs())
+ .anyMatch(JVMCommonUtils::hasSpaceWithoutRetry));
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
index 59bc5877979..1a51b78c452 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.disk.FolderManager;
import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType;
import org.apache.iotdb.commons.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.commons.utils.JVMCommonUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
@@ -58,20 +59,12 @@ import java.util.stream.Collectors;
public class TierManager {
private static final Logger logger =
LoggerFactory.getLogger(TierManager.class);
private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
- private DirectoryStrategyType directoryStrategyType =
DirectoryStrategyType.SEQUENCE_STRATEGY;
- /**
- * seq folder manager of each storage tier, managing both data directories
and multi-dir strategy
- */
- private volatile List<FolderManager> seqTiers = new ArrayList<>();
+ private record TierDirectory(int tierLevel, String folderName) {}
- /**
- * unSeq folder manager of each storage tier, managing both data directories
and multi-dir
- * strategy
- */
- private volatile List<FolderManager> unSeqTiers = new ArrayList<>();
-
- private volatile List<FolderManager> objectTiers = new ArrayList<>();
+ private volatile Map<TierDirectory, TierFolderManager> folderManagers =
Map.of();
+ // A failed constructor can report DiskFull before the replacement managers
are published.
+ private volatile boolean rebuildingFolders;
/** seq file folder's rawFsPath path -> tier level */
private volatile Map<String, Integer> seqDir2TierLevel = new HashMap<>();
@@ -81,31 +74,28 @@ public class TierManager {
private List<String> objectDirs;
- private List<String> copyToTargetDirs;
-
- // private FolderManager copyToFolderManager;
-
- private MemoizedCheckedSupplier<FolderManager,
DiskSpaceInsufficientException>
- copyToFolderManager;
-
/** total space of each tier, Long.MAX_VALUE when one tier contains remote
storage */
- private long[] tierDiskTotalSpace;
+ private volatile long[] tierDiskTotalSpace = new long[0];
private TierManager() {
initFolders();
}
public synchronized void initFolders() {
- initFolders(seqTiers, unSeqTiers, objectTiers, seqDir2TierLevel,
unSeqDir2TierLevel);
+
CommonDescriptor.getInstance().getConfig().invalidateDiskFullRecoverySnapshot();
+ rebuildingFolders = true;
+ try {
+ rebuildFolders();
+ } finally {
+ rebuildingFolders = false;
+ }
}
- private void initFolders(
- List<FolderManager> seqTiers,
- List<FolderManager> unSeqTiers,
- List<FolderManager> objectTiers,
- Map<String, Integer> seqDir2TierLevel,
- Map<String, Integer> unSeqDir2TierLevel) {
- directoryStrategyType =
+ private void rebuildFolders() {
+ Map<TierDirectory, TierFolderManager> newManagers = new HashMap<>();
+ Map<String, Integer> newSeqDir2TierLevel = new HashMap<>();
+ Map<String, Integer> newUnSeqDir2TierLevel = new HashMap<>();
+ DirectoryStrategyType directoryStrategyType =
DirectoryStrategyType.fromClassName(config.getMultiDirStrategyClassName());
config.updatePath();
@@ -140,13 +130,14 @@ public class TierManager {
.getPath())
.collect(Collectors.toList());
mkDataDirs(seqDirs);
- try {
- seqTiers.add(new FolderManager(seqDirs, directoryStrategyType));
- } catch (DiskSpaceInsufficientException e) {
- logger.error(StorageEngineMessages.ALL_DISKS_OF_TIER_FULL, tierLevel,
e);
- }
+ addFolderManager(
+ newManagers,
+ tierLevel,
+ IoTDBConstant.SEQUENCE_FOLDER_NAME,
+ seqDirs,
+ directoryStrategyType);
for (String dir : seqDirs) {
- seqDir2TierLevel.put(dir, tierLevel);
+ newSeqDir2TierLevel.put(dir, tierLevel);
}
List<String> unSeqDirs =
@@ -159,17 +150,18 @@ public class TierManager {
.getPath())
.collect(Collectors.toList());
mkDataDirs(unSeqDirs);
- try {
- unSeqTiers.add(new FolderManager(unSeqDirs, directoryStrategyType));
- } catch (DiskSpaceInsufficientException e) {
- logger.error(StorageEngineMessages.ALL_DISKS_OF_TIER_FULL, tierLevel,
e);
- }
+ addFolderManager(
+ newManagers,
+ tierLevel,
+ IoTDBConstant.UNSEQUENCE_FOLDER_NAME,
+ unSeqDirs,
+ directoryStrategyType);
for (String dir : unSeqDirs) {
- unSeqDir2TierLevel.put(dir, tierLevel);
+ newUnSeqDir2TierLevel.put(dir, tierLevel);
}
if (tierLevel == 0) {
- copyToTargetDirs =
+ List<String> copyToTargetDirs =
Arrays.stream(tierDirs[tierLevel])
.filter(Objects::nonNull)
.map(
@@ -178,9 +170,12 @@ public class TierManager {
.getFile(v,
IoTDBConstant.COPY_TO_TARGET_FOLDER_NAME)
.getPath())
.collect(Collectors.toList());
- copyToFolderManager =
- MemoizedCheckedSupplier.valueOf(
- () -> new FolderManager(copyToTargetDirs,
directoryStrategyType));
+ addFolderManager(
+ newManagers,
+ tierLevel,
+ IoTDBConstant.COPY_TO_TARGET_FOLDER_NAME,
+ copyToTargetDirs,
+ directoryStrategyType);
}
objectDirs =
@@ -193,11 +188,12 @@ public class TierManager {
.getPath())
.collect(Collectors.toList());
- try {
- objectTiers.add(new FolderManager(objectDirs, directoryStrategyType));
- } catch (DiskSpaceInsufficientException e) {
- logger.error(StorageEngineMessages.ALL_DISKS_OF_TIER_FULL, tierLevel,
e);
- }
+ addFolderManager(
+ newManagers,
+ tierLevel,
+ IoTDBConstant.OBJECT_FOLDER_NAME,
+ objectDirs,
+ directoryStrategyType);
// try to remove empty objectDirs
for (String dir : objectDirs) {
File dirFile = FSFactoryProducer.getFSFactory().getFile(dir);
@@ -211,23 +207,74 @@ public class TierManager {
}
tierDiskTotalSpace = getTierDiskSpace(DiskSpaceType.TOTAL);
+ seqDir2TierLevel = newSeqDir2TierLevel;
+ unSeqDir2TierLevel = newUnSeqDir2TierLevel;
+ folderManagers = newManagers;
+ }
+
+ private void addFolderManager(
+ Map<TierDirectory, TierFolderManager> newManagers,
+ int tierLevel,
+ String folderName,
+ List<String> folders,
+ DirectoryStrategyType type) {
+ TierDirectory key = new TierDirectory(tierLevel, folderName);
+ TierFolderManager entry = new TierFolderManager(key, folders, type);
+ newManagers.put(key, entry);
+ if (!IoTDBConstant.COPY_TO_TARGET_FOLDER_NAME.equals(folderName)) {
+ try {
+ entry.supplier.get();
+ } catch (DiskSpaceInsufficientException e) {
+ logger.error(StorageEngineMessages.ALL_DISKS_OF_TIER_FULL, tierLevel,
e);
+ }
+ }
+ }
+
+ private final class TierFolderManager {
+ private final List<String> folders;
+ private final MemoizedCheckedSupplier<FolderManager,
DiskSpaceInsufficientException> supplier;
+ private volatile FolderManager manager;
+
+ private TierFolderManager(TierDirectory key, List<String> folders,
DirectoryStrategyType type) {
+ this.folders = folders;
+ supplier =
+ MemoizedCheckedSupplier.valueOf(
+ () -> {
+ manager = new FolderManager(folders, type, () ->
reportDiskFull(key, this));
+ return manager;
+ });
+ }
+
+ private boolean hasSpaceForRecovery() {
+ FolderManager current = manager;
+ return current == null
+ ? folders.stream().anyMatch(JVMCommonUtils::hasSpaceWithoutRetry)
+ : current.hasSpaceForRecovery();
+ }
+ }
+
+ private void reportDiskFull(TierDirectory key, TierFolderManager source) {
+ if (!rebuildingFolders && folderManagers.get(key) != source) {
+ return;
+ }
+ CommonDescriptor.getInstance()
+ .getConfig()
+ .setReadOnlyForDiskFull(key, () -> hasSpaceForRecovery(key));
+ }
+
+ private boolean hasSpaceForRecovery(TierDirectory key) {
+ Map<TierDirectory, TierFolderManager> active = folderManagers;
+ if (rebuildingFolders) {
+ return false;
+ }
+ TierFolderManager current = active.get(key);
+ boolean recovered = current == null || current.hasSpaceForRecovery();
+ return recovered && active == folderManagers && !rebuildingFolders;
}
public synchronized void resetFolders() {
long startTime = System.currentTimeMillis();
- List<FolderManager> newSeqTiers = new ArrayList<>();
- List<FolderManager> newUnSeqTiers = new ArrayList<>();
- List<FolderManager> newObjectTiers = new ArrayList<>();
- Map<String, Integer> newSeqDir2TierLevel = new HashMap<>();
- Map<String, Integer> newUnSeqDir2TierLevel = new HashMap<>();
-
- initFolders(
- newSeqTiers, newUnSeqTiers, newObjectTiers, newSeqDir2TierLevel,
newUnSeqDir2TierLevel);
- seqTiers = newSeqTiers;
- unSeqTiers = newUnSeqTiers;
- objectTiers = newObjectTiers;
- seqDir2TierLevel = newSeqDir2TierLevel;
- unSeqDir2TierLevel = newUnSeqDir2TierLevel;
+ initFolders();
long endTime = System.currentTimeMillis();
logger.info(StorageEngineMessages.FOLDERS_RESET_SUCCESSFULLY, (endTime -
startTime));
}
@@ -251,21 +298,32 @@ public class TierManager {
public String getNextFolderForTsFile(int tierLevel, boolean sequence)
throws DiskSpaceInsufficientException {
- return sequence
- ? seqTiers.get(tierLevel).getNextFolder()
- : unSeqTiers.get(tierLevel).getNextFolder();
+ return getFolderManager(tierLevel, sequence).getNextFolder();
}
public String getNextFolderForCopyToTargetFile() throws
DiskSpaceInsufficientException {
- return copyToFolderManager.get().getNextFolder();
+ return getFolderManager(0,
IoTDBConstant.COPY_TO_TARGET_FOLDER_NAME).getNextFolder();
}
public String getNextFolderForObjectFile() throws
DiskSpaceInsufficientException {
- return objectTiers.get(0).getNextFolder();
+ return getFolderManager(0,
IoTDBConstant.OBJECT_FOLDER_NAME).getNextFolder();
+ }
+
+ public FolderManager getFolderManager(int tierLevel, boolean sequence)
+ throws DiskSpaceInsufficientException {
+ String folderName =
+ sequence ? IoTDBConstant.SEQUENCE_FOLDER_NAME :
IoTDBConstant.UNSEQUENCE_FOLDER_NAME;
+ return getFolderManager(tierLevel, folderName);
}
- public FolderManager getFolderManager(int tierLevel, boolean sequence) {
- return sequence ? seqTiers.get(tierLevel) : unSeqTiers.get(tierLevel);
+ private FolderManager getFolderManager(int tierLevel, String folderName)
+ throws DiskSpaceInsufficientException {
+ TierDirectory key = new TierDirectory(tierLevel, folderName);
+ TierFolderManager entry = folderManagers.get(key);
+ if (entry == null) {
+ throw new IndexOutOfBoundsException(key.toString());
+ }
+ return entry.supplier.get();
}
public List<String> getAllFilesFolders() {
@@ -356,7 +414,7 @@ public class TierManager {
}
public int getTiersNum() {
- return seqTiers.size();
+ return tierDiskTotalSpace.length;
}
public int getFileTierLevel(File file) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DiskFullHotReloadTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DiskFullHotReloadTest.java
index 38e1a44d551..620f7e1ec7e 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DiskFullHotReloadTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/conf/DiskFullHotReloadTest.java
@@ -39,7 +39,6 @@ import java.util.Collections;
import java.util.Map;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
public class DiskFullHotReloadTest {
@@ -74,20 +73,20 @@ public class DiskFullHotReloadTest {
}
@Test
- public void loadSameConfigurationClearsOldDiskFullChecks() throws Exception {
+ public void loadSameConfigurationKeepsUnrecoveredDiskFullChecks() throws
Exception {
config.setReadOnlyForDiskFull("old directory", () -> false);
descriptor.loadHotModifiedProps();
- assertRecoveredAndCleared();
+ assertStillProtected();
}
@Test
- public void setUnrelatedPropertyClearsOldDiskFullChecks() {
+ public void setUnrelatedPropertyKeepsUnrecoveredDiskFullChecks() {
config.setReadOnlyForDiskFull("old directory", () -> false);
Map<String, String> update =
Map.of("enable_partial_insert",
Boolean.toString(!originalPartialInsert));
assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(),
setConfiguration(update));
assertEquals(!originalPartialInsert,
descriptor.getConfig().isEnablePartialInsert());
- assertRecoveredAndCleared();
+ assertStillProtected();
}
@Test
@@ -131,14 +130,10 @@ public class DiskFullHotReloadTest {
.getCode();
}
- private void assertRecoveredAndCleared() {
- assertEquals(NodeStatus.Running, config.getNodeStatus());
- assertNull(config.getStatusReason());
- // A later aggregate-only failure must have no callback left from the
previous failure.
- config.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL);
+ private void assertStillProtected() {
+ assertReadOnly(NodeStatus.DISK_FULL);
config.tryRecoverFromDiskFull();
- assertEquals(NodeStatus.Running, config.getNodeStatus());
- assertNull(config.getStatusReason());
+ assertReadOnly(NodeStatus.DISK_FULL);
}
private void assertReadOnly(String reason) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoaderDiskFullReloadTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoaderDiskFullReloadTest.java
new file mode 100644
index 00000000000..d8d6b38d8a6
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoaderDiskFullReloadTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.snapshot;
+
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.commons.utils.JVMCommonUtils;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*",
"javax.management.*"})
+@PrepareForTest(JVMCommonUtils.class)
+public class SnapshotLoaderDiskFullReloadTest {
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ private final CommonConfig commonConfig =
CommonDescriptor.getInstance().getConfig();
+ private String[][] originalTierDataDirs;
+ private NodeStatus originalStatus;
+ private String originalReason;
+ private TierManager tierManager;
+
+ @Before
+ public void setUp() {
+ originalTierDataDirs = config.getTierDataDirs();
+ originalStatus = commonConfig.getNodeStatus();
+ originalReason = commonConfig.getStatusReason();
+ config.setTierDataDirs(
+ new String[][] {{new File(temporaryFolder.getRoot(),
"old").getAbsolutePath()}});
+ tierManager = TierManager.getInstance();
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ PowerMockito.mockStatic(JVMCommonUtils.class);
+ }
+
+ @After
+ public void tearDown() {
+ when(JVMCommonUtils.hasSpace(anyString())).thenReturn(true);
+ config.setTierDataDirs(originalTierDataDirs);
+ tierManager.resetFolders();
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ commonConfig.setNodeStatusWithReason(originalStatus, originalReason);
+ }
+
+ @Test
+ public void testFailedSnapshotManagerRecoversAfterAddingDataDir() throws
Exception {
+ File snapshotDir = temporaryFolder.newFolder("snapshot");
+ assertTrue(new File(snapshotDir, "sequence/root.sg/0").mkdirs());
+ SnapshotLoader loader = new SnapshotLoader(snapshotDir.getAbsolutePath(),
"root.sg", "0");
+ assertThrows(
+ DiskSpaceInsufficientException.class,
+ () ->
+ loader.createLinksFromSnapshotDirToDataDirWithoutLog(
+ snapshotDir, new HashMap<>(), false));
+ assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+
+ AtomicBoolean walAvailable = new AtomicBoolean();
+ commonConfig.setReadOnlyForDiskFull("wal", walAvailable::get);
+ File oldDir = new File(temporaryFolder.getRoot(), "old");
+ File newDir = new File(temporaryFolder.getRoot(), "new");
+ config.setTierDataDirs(new String[][] {{oldDir.getAbsolutePath(),
newDir.getAbsolutePath()}});
+
when(JVMCommonUtils.hasSpaceWithoutRetry(newDir.getCanonicalPath())).thenReturn(true);
+ for (String folderName :
+ new String[] {
+ IoTDBConstant.SEQUENCE_FOLDER_NAME,
+ IoTDBConstant.UNSEQUENCE_FOLDER_NAME,
+ IoTDBConstant.OBJECT_FOLDER_NAME
+ }) {
+ String folder = new File(newDir, folderName).getCanonicalPath();
+ when(JVMCommonUtils.hasSpace(folder)).thenReturn(true);
+ when(JVMCommonUtils.hasSpaceWithoutRetry(folder)).thenReturn(true);
+ }
+ tierManager.resetFolders();
+
+ commonConfig.tryRecoverFromDiskFull();
+ assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ walAvailable.set(true);
+ commonConfig.tryRecoverFromDiskFull();
+ assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ assertEquals(
+ new File(newDir,
IoTDBConstant.SEQUENCE_FOLDER_NAME).getCanonicalPath(),
+ tierManager.getNextFolderForTsFile(0, true));
+ }
+}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManagerDiskFullReloadTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManagerDiskFullReloadTest.java
new file mode 100644
index 00000000000..9fd46374039
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManagerDiskFullReloadTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.storageengine.rescon.disk;
+
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.utils.JVMCommonUtils;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.File;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*",
"javax.management.*"})
+@PrepareForTest(JVMCommonUtils.class)
+public class TierManagerDiskFullReloadTest {
+
+ @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ private final CommonConfig commonConfig =
CommonDescriptor.getInstance().getConfig();
+ private String[][] originalTierDataDirs;
+ private NodeStatus originalStatus;
+ private String originalReason;
+ private TierManager tierManager;
+ private File oldDir;
+ private File newDir;
+
+ @Before
+ public void setUp() {
+ originalTierDataDirs = config.getTierDataDirs();
+ originalStatus = commonConfig.getNodeStatus();
+ originalReason = commonConfig.getStatusReason();
+ oldDir = new File(temporaryFolder.getRoot(), "old");
+ newDir = new File(temporaryFolder.getRoot(), "new");
+ config.setTierDataDirs(new String[][] {{oldDir.getAbsolutePath()}});
+ tierManager = TierManager.getInstance();
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ PowerMockito.mockStatic(JVMCommonUtils.class);
+ }
+
+ @After
+ public void tearDown() {
+ when(JVMCommonUtils.hasSpace(anyString())).thenReturn(true);
+ config.setTierDataDirs(originalTierDataDirs);
+ tierManager.resetFolders();
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ commonConfig.setNodeStatusWithReason(originalStatus, originalReason);
+ }
+
+ @Test
+ public void testNewDataDirectoryDoesNotClearAnotherFullSource() throws
Exception {
+ tierManager.resetFolders();
+ assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+
+ tierManager.resetFolders();
+ commonConfig.tryRecoverFromDiskFull();
+ assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+
+ AtomicBoolean walAvailable = new AtomicBoolean();
+ commonConfig.setReadOnlyForDiskFull("wal", walAvailable::get);
+ config.setTierDataDirs(new String[][] {{oldDir.getAbsolutePath(),
newDir.getAbsolutePath()}});
+ for (String folderName :
+ new String[] {
+ IoTDBConstant.SEQUENCE_FOLDER_NAME,
+ IoTDBConstant.UNSEQUENCE_FOLDER_NAME,
+ IoTDBConstant.OBJECT_FOLDER_NAME
+ }) {
+ String folder = new File(newDir, folderName).getCanonicalPath();
+ when(JVMCommonUtils.hasSpace(folder)).thenReturn(true);
+ when(JVMCommonUtils.hasSpaceWithoutRetry(folder)).thenReturn(true);
+ }
+ tierManager.resetFolders();
+
+ commonConfig.tryRecoverFromDiskFull();
+ assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ assertEquals(NodeStatus.DISK_FULL, commonConfig.getStatusReason());
+
+ walAvailable.set(true);
+ commonConfig.tryRecoverFromDiskFull();
+ assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ assertEquals(
+ new File(newDir,
IoTDBConstant.SEQUENCE_FOLDER_NAME).getCanonicalPath(),
+ tierManager.getNextFolderForTsFile(0, true));
+ }
+}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManagerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManagerTest.java
index af800b81df6..defedd56b89 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManagerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManagerTest.java
@@ -19,7 +19,12 @@
package org.apache.iotdb.db.storageengine.rescon.disk;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.commons.utils.JVMCommonUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -41,7 +46,9 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
public class TierManagerTest {
@@ -141,6 +148,49 @@ public class TierManagerTest {
}
}
+ @Test
+ public void testInitiallyFullTiersRecoverWithoutLosingTierLevels() throws
Exception {
+ CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig();
+ NodeStatus originalStatus = commonConfig.getNodeStatus();
+ String originalReason = commonConfig.getStatusReason();
+ double originalThreshold = commonConfig.getDiskSpaceWarningThreshold();
+ String tier0 = new File(temporaryFolder.getRoot(),
"tier0").getAbsolutePath();
+ String tier1 = new File(temporaryFolder.getRoot(),
"tier1").getAbsolutePath();
+
+ try {
+ config.setTierDataDirs(new String[][] {{tier0}, {tier1}});
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ JVMCommonUtils.setDiskSpaceWarningThreshold(1.0);
+ tierManager.resetFolders();
+
+ assertEquals(2, tierManager.getTiersNum());
+ assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ IndexOutOfBoundsException missingTier =
+ assertThrows(
+ IndexOutOfBoundsException.class, () ->
tierManager.getFolderManager(2, true));
+ assertTrue(missingTier.getMessage().contains("tierLevel=2"));
+ assertTrue(missingTier.getMessage().contains("folderName=sequence"));
+ assertThrows(
+ DiskSpaceInsufficientException.class, () ->
tierManager.getFolderManager(0, true));
+ assertThrows(
+ DiskSpaceInsufficientException.class, () ->
tierManager.getFolderManager(1, false));
+
+ JVMCommonUtils.setDiskSpaceWarningThreshold(0.0);
+ commonConfig.tryRecoverFromDiskFull();
+ assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ assertEquals(
+ new File(tier0,
IoTDBConstant.SEQUENCE_FOLDER_NAME).getCanonicalPath(),
+ tierManager.getNextFolderForTsFile(0, true));
+ assertEquals(
+ new File(tier1,
IoTDBConstant.UNSEQUENCE_FOLDER_NAME).getCanonicalPath(),
+ tierManager.getNextFolderForTsFile(1, false));
+ } finally {
+ JVMCommonUtils.setDiskSpaceWarningThreshold(originalThreshold);
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ commonConfig.setNodeStatusWithReason(originalStatus, originalReason);
+ }
+ }
+
private List<String> assertFoldersAvailable(List<String> folders) {
assertTrue(folders.size() >= DATA_DIR_NUM);
return folders;
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index f3ba8c27b10..d2a92a958f4 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -189,6 +189,12 @@ public class CommonConfig {
// a live FolderManager uses its identity so its health restrictions are
also checked.
private final Map<Object, BooleanSupplier> diskFullRecoveryChecks = new
HashMap<>();
+ /**
+ * Invalidates a recovery check that was evaluated before another disk
failure or directory
+ * reload.
+ */
+ private long diskFullRecoveryVersion;
+
private final int TTimePartitionSlotTransmitLimit = 1000;
/** Disk Monitor. */
@@ -853,29 +859,32 @@ public class CommonConfig {
setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL);
}
- public synchronized boolean isReadOnlyForDiskFull() {
- return status == NodeStatus.ReadOnly &&
NodeStatus.DISK_FULL.equals(statusReason);
+ public synchronized void invalidateDiskFullRecoverySnapshot() {
+ diskFullRecoveryVersion++;
}
- /** Recheck the reason under the status lock before clearing disk-full
recovery checks. */
- public synchronized void setRunningIfDiskFull() {
- if (isReadOnlyForDiskFull()) {
- setNodeStatus(NodeStatus.Running);
- }
+ public synchronized boolean isReadOnlyForDiskFull() {
+ return status == NodeStatus.ReadOnly &&
NodeStatus.DISK_FULL.equals(statusReason);
}
/** Called after the aggregate disk-space check passes. */
public void tryRecoverFromDiskFull() {
final List<BooleanSupplier> recoveryChecks;
+ final long recoveryVersion;
synchronized (this) {
if (!isReadOnlyForDiskFull()) {
return;
}
recoveryChecks = new ArrayList<>(diskFullRecoveryChecks.values());
+ recoveryVersion = diskFullRecoveryVersion;
}
// Do not hold the status monitor during filesystem I/O.
if (recoveryChecks.stream().allMatch(BooleanSupplier::getAsBoolean)) {
- setRunningIfDiskFull();
+ synchronized (this) {
+ if (recoveryVersion == diskFullRecoveryVersion &&
isReadOnlyForDiskFull()) {
+ setNodeStatus(NodeStatus.Running);
+ }
+ }
}
}
@@ -885,6 +894,9 @@ public class CommonConfig {
* reason.
*/
public synchronized void setNodeStatusWithReason(NodeStatus newStatus,
String newReason) {
+ if (newStatus == NodeStatus.ReadOnly &&
NodeStatus.DISK_FULL.equals(newReason)) {
+ diskFullRecoveryVersion++;
+ }
if (newStatus != NodeStatus.ReadOnly) {
diskFullRecoveryChecks.clear();
newReason = null;
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
index 1cb969043e9..4dee48cc47c 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
@@ -19,7 +19,6 @@
package org.apache.iotdb.commons.disk;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.disk.strategy.DirectoryStrategy;
import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType;
import org.apache.iotdb.commons.disk.strategy.MaxDiskUsableSpaceFirstStrategy;
@@ -75,6 +74,21 @@ public class FolderManager {
public FolderManager(
List<String> folders, DirectoryStrategyType type, boolean
changeSystemStatusToReadOnly)
throws DiskSpaceInsufficientException {
+ this(folders, type, changeSystemStatusToReadOnly, null);
+ }
+
+ /** For an owner that replaces this manager on configuration reload. */
+ public FolderManager(List<String> folders, DirectoryStrategyType type,
Runnable diskFullHandler)
+ throws DiskSpaceInsufficientException {
+ this(folders, type, true, diskFullHandler);
+ }
+
+ private FolderManager(
+ List<String> folders,
+ DirectoryStrategyType type,
+ boolean changeSystemStatusToReadOnly,
+ Runnable diskFullHandler)
+ throws DiskSpaceInsufficientException {
this.folders = folders;
this.changeSystemStatusToReadOnly = changeSystemStatusToReadOnly;
folders.forEach(dir -> foldersStates.put(dir, FolderState.HEALTHY));
@@ -95,6 +109,7 @@ public class FolderManager {
throw new RuntimeException();
}
this.selectStrategy.setChangeSystemStatusToReadOnly(changeSystemStatusToReadOnly);
+ this.selectStrategy.setDiskFullHandler(diskFullHandler);
this.selectStrategy.setFolders(folders);
this.selectStrategy.setFoldersStates(foldersStates);
}
@@ -128,6 +143,10 @@ public class FolderManager {
&& hasSpace.test(folder));
}
+ public boolean hasSpaceForRecovery() {
+ return
hasFolderWithAvailableDiskSpace(JVMCommonUtils::hasSpaceWithoutRetry);
+ }
+
private void changeToReadOnlyIfDiskFull(DiskSpaceInsufficientException e) {
if (!changeSystemStatusToReadOnly) {
return;
@@ -137,10 +156,7 @@ public class FolderManager {
if
(LoggerPeriodicalLogReducer.shouldLog(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY))
{
logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e);
}
- CommonDescriptor.getInstance()
- .getConfig()
- .setReadOnlyForDiskFull(
- this, () ->
hasFolderWithAvailableDiskSpace(JVMCommonUtils::hasSpaceWithoutRetry));
+ selectStrategy.reportDiskFull(this, this::hasSpaceForRecovery);
} else {
logger.warn(UtilMessages.CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE, e);
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java
index a592766ec78..4e1072d07d4 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/DirectoryStrategy.java
@@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.BooleanSupplier;
import static org.apache.iotdb.commons.disk.FolderManager.FolderState.HEALTHY;
@@ -48,6 +49,20 @@ public abstract class DirectoryStrategy {
private boolean changeSystemStatusToReadOnly = true;
+ private Runnable diskFullHandler;
+
+ public void setDiskFullHandler(Runnable diskFullHandler) {
+ this.diskFullHandler = diskFullHandler;
+ }
+
+ public void reportDiskFull(Object source, BooleanSupplier recoveryCheck) {
+ if (diskFullHandler != null) {
+ diskFullHandler.run();
+ } else {
+
CommonDescriptor.getInstance().getConfig().setReadOnlyForDiskFull(source,
recoveryCheck);
+ }
+ }
+
public void setChangeSystemStatusToReadOnly(boolean
changeSystemStatusToReadOnly) {
this.changeSystemStatusToReadOnly = changeSystemStatusToReadOnly;
}
@@ -70,11 +85,8 @@ public abstract class DirectoryStrategy {
if (changeSystemStatusToReadOnly) {
LOGGER.error(UtilMessages.DISK_SPACE_INSUFFICIENT_READ_ONLY);
final List<String> fullFolders = List.copyOf(folders);
- CommonDescriptor.getInstance()
- .getConfig()
- .setReadOnlyForDiskFull(
- fullFolders,
- () ->
fullFolders.stream().anyMatch(JVMCommonUtils::hasSpaceWithoutRetry));
+ reportDiskFull(
+ fullFolders, () ->
fullFolders.stream().anyMatch(JVMCommonUtils::hasSpaceWithoutRetry));
} else {
if (LoggerPeriodicalLogReducer.shouldLog(
UtilMessages.MESSAGE_DISK_SPACE_INSUFFICIENT_DF6205B0)) {
diff --git
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java
index 494db7d94ed..95fc05351b7 100644
---
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java
+++
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonConfigTest.java
@@ -254,6 +254,50 @@ public class CommonConfigTest {
}
}
+ @Test
+ public void testDiskRecoveryCannotClearNewFailureDuringRecoveryCheck() {
+ CommonConfig config = new CommonConfig();
+ AtomicBoolean reportedNewFailure = new AtomicBoolean();
+ AtomicBoolean newSourceAvailable = new AtomicBoolean();
+ config.setReadOnlyForDiskFull(
+ "first",
+ () -> {
+ if (reportedNewFailure.compareAndSet(false, true)) {
+ config.setReadOnlyForDiskFull("second", newSourceAvailable::get);
+ }
+ return true;
+ });
+
+ config.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL, config.getStatusReason());
+
+ newSourceAvailable.set(true);
+ config.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.Running, config.getNodeStatus());
+ }
+
+ @Test
+ public void testDiskRecoveryCannotClearDirectoryReloadDuringRecoveryCheck() {
+ CommonConfig config = new CommonConfig();
+ AtomicBoolean reloaded = new AtomicBoolean();
+ config.setReadOnlyForDiskFull(
+ "tier",
+ () -> {
+ if (reloaded.compareAndSet(false, true)) {
+ config.invalidateDiskFullRecoverySnapshot();
+ }
+ return true;
+ });
+
+ config.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.ReadOnly, config.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL, config.getStatusReason());
+
+ config.tryRecoverFromDiskFull();
+ Assert.assertEquals(NodeStatus.Running, config.getNodeStatus());
+ }
+
@Test
public void testManualRunningClearsPreviousDiskRecoveryChecks() {
CommonConfig config = new CommonConfig();