This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9eb39bee848 Fix false disk-full read-only during concurrent snapshot
directory deletion (#18114)
9eb39bee848 is described below
commit 9eb39bee84862dbb150053096194daab3e26b7e1
Author: Hongzhi Gao <[email protected]>
AuthorDate: Mon Jul 6 12:16:27 2026 +0800
Fix false disk-full read-only during concurrent snapshot directory deletion
(#18114)
When receiving snapshot fragments, getOccupiedSpace walks the shared
snapshot root while other regions may delete their staging subdirectories
concurrently. NoSuchFileException was misinterpreted as folder full, triggering
node read-only and failed region migration during datanode removal.
---
.../apache/iotdb/commons/i18n/UtilMessages.java | 2 +
.../apache/iotdb/commons/i18n/UtilMessages.java | 2 +
.../apache/iotdb/commons/disk/FolderManager.java | 30 +++++++----
.../MinFolderOccupiedSpaceFirstStrategy.java | 3 +-
.../apache/iotdb/commons/utils/JVMCommonUtils.java | 30 +++++++----
...FolderOccupiedSpaceFirstStrategyRealFsTest.java | 61 ++++++++++++++++++++++
.../iotdb/commons/utils/JVMCommonUtilsTest.java | 48 +++++++++++++++++
7 files changed, 155 insertions(+), 21 deletions(-)
diff --git
a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java
b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java
index 6008e507a9e..0fbf9a941e5 100644
---
a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java
+++
b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java
@@ -112,6 +112,8 @@ public final class UtilMessages {
public static final String ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY =
"All folders are full, change system mode to read-only.";
+ public static final String CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE =
+ "Cannot select folder but disk still has available space, will not
change to read-only.";
public static final String FAILED_TO_PROCESS_FOLDER = "Failed to process
folder {}";
public static final String FAILED_TO_READ_FILE_STORE_PATH =
"Failed to read file store path '{}'";
diff --git
a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java
b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java
index e1cfb9f7305..6cdf3ffaacb 100644
---
a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java
+++
b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java
@@ -110,6 +110,8 @@ public final class UtilMessages {
public static final String ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY =
"所有文件夹空间均已耗尽，系统切换为只读模式。";
+ public static final String CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE =
+ "无法选择文件夹但磁盘仍有可用空间，不切换为只读模式。";
public static final String FAILED_TO_PROCESS_FOLDER = "处理文件夹 {} 失败";
public static final String FAILED_TO_READ_FILE_STORE_PATH =
"读取文件存储路径 '{}' 失败";
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
index 731dc66fa4a..08e2b7872ef 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/FolderManager.java
@@ -88,9 +88,7 @@ public class FolderManager {
this.selectStrategy.setFolders(folders);
this.selectStrategy.setFoldersStates(foldersStates);
} catch (DiskSpaceInsufficientException e) {
- logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e);
-
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
-
CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL);
+ changeToReadOnlyIfDiskFull(e);
throw e;
}
}
@@ -104,9 +102,7 @@ public class FolderManager {
try {
return folders.get(selectStrategy.nextFolderIndex());
} catch (DiskSpaceInsufficientException e) {
- logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e);
-
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
-
CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL);
+ changeToReadOnlyIfDiskFull(e);
throw e;
}
}
@@ -118,6 +114,24 @@ public class FolderManager {
foldersStates.getOrDefault(folder, FolderState.ABNORMAL) ==
FolderState.HEALTHY);
}
+ private boolean hasFolderWithAvailableDiskSpace() {
+ return folders.stream()
+ .anyMatch(
+ folder ->
+ foldersStates.getOrDefault(folder, FolderState.ABNORMAL) ==
FolderState.HEALTHY
+ && JVMCommonUtils.hasSpace(folder));
+ }
+
+ private void changeToReadOnlyIfDiskFull(DiskSpaceInsufficientException e) {
+ if (!hasFolderWithAvailableDiskSpace()) {
+ logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e);
+
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
+
CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL);
+ } else {
+ logger.warn(UtilMessages.CANNOT_SELECT_FOLDER_BUT_DISK_HAS_SPACE, e);
+ }
+ }
+
@FunctionalInterface
public interface ThrowingFunction<T, R, E extends Exception> {
R apply(T t) throws E;
@@ -135,9 +149,7 @@ public class FolderManager {
try {
folder = folders.get(selectStrategy.nextFolderIndex());
} catch (DiskSpaceInsufficientException e) {
- logger.error(UtilMessages.ALL_FOLDERS_FULL_CHANGE_TO_READ_ONLY, e);
-
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
-
CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL);
+ changeToReadOnlyIfDiskFull(e);
throw e;
}
try {
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java
index 6a893d47aa1..b49358e4ada 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java
@@ -120,8 +120,7 @@ public class MinFolderOccupiedSpaceFirstStrategy extends
DirectoryStrategy {
try {
cachedOccupiedSpace[i] = JVMCommonUtils.getOccupiedSpace(folder);
} catch (IOException | UncheckedIOException e) {
- LOGGER.error(UtilMessages.CANNOT_CALCULATE_OCCUPIED_SPACE, folder, e);
- cachedOccupiedSpace[i] = Long.MAX_VALUE;
+ LOGGER.warn(UtilMessages.CANNOT_CALCULATE_OCCUPIED_SPACE, folder, e);
}
}
selectionsSinceRefresh = 0;
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
index c421b3d7527..8ef3fe561c7 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
@@ -28,10 +28,12 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.stream.Stream;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
public class JVMCommonUtils {
@@ -110,15 +112,23 @@ public class JVMCommonUtils {
if (!Files.exists(folder)) {
return 0;
}
- try (Stream<Path> s = Files.walk(folder)) {
- return s.filter(p -> p.toFile().isFile())
- .mapToLong(
- p -> {
- File file = p.toFile();
- return file.exists() ? file.length() : 0L;
- })
- .sum();
- }
+ final long[] occupiedSpace = {0L};
+ Files.walkFileTree(
+ folder,
+ new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs) {
+ occupiedSpace[0] += attrs.size();
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFileFailed(Path file, IOException exc) {
+ // A file or directory may be deleted concurrently during
traversal; ignore it.
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ return occupiedSpace[0];
}
public static int getCpuCores() {
diff --git
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java
index 6f4797233c5..d255d6fb7f4 100644
---
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java
+++
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/disk/MinFolderOccupiedSpaceFirstStrategyRealFsTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.commons.disk;
+import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType;
import
org.apache.iotdb.commons.disk.strategy.MinFolderOccupiedSpaceFirstStrategy;
@@ -34,8 +35,10 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Stream;
import static org.junit.Assert.assertEquals;
@@ -124,4 +127,62 @@ public class MinFolderOccupiedSpaceFirstStrategyRealFsTest
{
// and correctly avoids the now-largest folder 0, picking the least
occupied folder 1.
assertEquals(1, strategy.nextFolderIndex());
}
+
+ @Test
+ public void getNextFolderDoesNotEnterReadOnlyWhenDiskHasSpace() throws
Exception {
+ Path snapshotRoot = Files.createTempDirectory("snapshot-root-");
+ tempDirs.add(snapshotRoot);
+ Path subDir = snapshotRoot.resolve("tod_sod0-71");
+ Files.createDirectories(subDir);
+ writeFileInPath(subDir, "a.bin", 1024);
+
+ List<String> singleFolder =
Collections.singletonList(snapshotRoot.toFile().getAbsolutePath());
+
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Running);
+
+ AtomicBoolean stop = new AtomicBoolean(false);
+ Thread deleter =
+ new Thread(
+ () -> {
+ while (!stop.get()) {
+ try {
+ deleteRecursively(subDir);
+ Files.createDirectories(subDir);
+ writeFileInPath(subDir, "temp.bin", 512);
+ Thread.sleep(1);
+ } catch (Exception ignored) {
+ // Concurrent create/delete is expected in this test.
+ }
+ }
+ });
+ deleter.start();
+
+ try {
+ FolderManager folderManager =
+ new FolderManager(
+ singleFolder,
DirectoryStrategyType.MIN_FOLDER_OCCUPIED_SPACE_FIRST_STRATEGY);
+ for (int i = 0; i < 100; i++) {
+ assertEquals(
+ NodeStatus.Running,
CommonDescriptor.getInstance().getConfig().getNodeStatus());
+ assertEquals(singleFolder.get(0), folderManager.getNextFolder());
+ }
+ } finally {
+ stop.set(true);
+ deleter.join(5000);
+ }
+ }
+
+ private void writeFileInPath(Path folder, String name, int sizeBytes) throws
IOException {
+ byte[] payload = new byte[sizeBytes];
+ Arrays.fill(payload, (byte) 1);
+ Files.write(folder.resolve(name), payload);
+ }
+
+ private static void deleteRecursively(Path path) throws IOException {
+ if (!Files.exists(path)) {
+ return;
+ }
+ try (Stream<Path> walk = Files.walk(path)) {
+
walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
+ }
+ }
}
diff --git
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java
index d35d3a52ea2..e314093d9f4 100644
---
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java
+++
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/JVMCommonUtilsTest.java
@@ -28,6 +28,10 @@ import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Stream;
public class JVMCommonUtilsTest {
@@ -66,4 +70,48 @@ public class JVMCommonUtilsTest {
Assert.assertEquals(
2L * payload.length,
JVMCommonUtils.getOccupiedSpace(dir.getAbsolutePath()));
}
+
+ @Test
+ public void getOccupiedSpaceIgnoresConcurrentlyDeletedEntries() throws
Exception {
+ File snapshotRoot = tempFolder.newFolder("snapshot");
+ File subDir = new File(snapshotRoot, "tod_sod0-71");
+ Assert.assertTrue(subDir.mkdirs());
+ byte[] payload = "snapshot-data".getBytes(StandardCharsets.UTF_8);
+ Files.write(new File(subDir, "a.bin").toPath(), payload);
+ Files.write(new File(snapshotRoot, "other.bin").toPath(), payload);
+
+ AtomicBoolean stop = new AtomicBoolean(false);
+ Thread deleter =
+ new Thread(
+ () -> {
+ while (!stop.get()) {
+ try {
+ deleteRecursively(subDir.toPath());
+ Files.createDirectories(subDir.toPath());
+ Files.write(new File(subDir, "temp.bin").toPath(), payload);
+ } catch (IOException ignored) {
+ // Concurrent create/delete is expected in this test.
+ }
+ }
+ });
+ deleter.start();
+
+ try {
+ for (int i = 0; i < 200; i++) {
+ JVMCommonUtils.getOccupiedSpace(snapshotRoot.getAbsolutePath());
+ }
+ } finally {
+ stop.set(true);
+ deleter.join(5000);
+ }
+ }
+
+ private static void deleteRecursively(Path path) throws IOException {
+ if (!Files.exists(path)) {
+ return;
+ }
+ try (Stream<Path> walk = Files.walk(path)) {
+
walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
+ }
+ }
}
