This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new f8e24dc71eb Feature/retry on disk failure (#15676)
f8e24dc71eb is described below
commit f8e24dc71eb3838aa149158c70354d114e9e3952
Author: Hongzhi Gao <[email protected]>
AuthorDate: Thu Jul 3 18:40:06 2025 +0800
Feature/retry on disk failure (#15676)
* Skip faulty disk
* fix some issues
* Adjusted the log messages related to disk failures.
* Adjusted the log messages related to disk failures.
* Format code using Spotless
* Added FolderManager.getNextWithRetry(Consumer folderConsumer)
* fix ut failures
---
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 3 +
.../iotdb/db/exception/load/LoadFileException.java | 4 ++
.../connector/util/builder/PipeTsFileBuilder.java | 40 +++++++-----
.../pipeconsensus/PipeConsensusReceiver.java | 75 +++++++++++-----------
.../protocol/thrift/IoTDBDataNodeReceiver.java | 5 ++
.../dataregion/snapshot/SnapshotLoader.java | 67 +++++++++++++------
.../tsfile/generator/TsFileNameGenerator.java | 63 ++++++++++--------
.../allocation/AbstractNodeAllocationStrategy.java | 13 ++--
.../db/storageengine/load/LoadTsFileManager.java | 12 ++--
.../storageengine/load/disk/ILoadDiskSelector.java | 3 +-
.../InheritSystemMultiDisksStrategySelector.java | 42 ++++++++----
.../db/storageengine/load/disk/MinIOSelector.java | 3 +-
.../storageengine/rescon/disk/FolderManager.java | 63 ++++++++++++++++++
.../db/storageengine/rescon/disk/TierManager.java | 4 ++
.../rescon/disk/strategy/DirectoryStrategy.java | 34 ++++++++++
.../strategy/MaxDiskUsableSpaceFirstStrategy.java | 3 +
.../MinFolderOccupiedSpaceFirstStrategy.java | 4 ++
.../strategy/RandomOnDiskUsableSpaceStrategy.java | 4 ++
.../rescon/disk/strategy/SequenceStrategy.java | 4 +-
.../commons/pipe/receiver/IoTDBFileReceiver.java | 66 ++++++++++---------
.../apache/iotdb/commons/utils/JVMCommonUtils.java | 56 +++++++++-------
21 files changed, 389 insertions(+), 179 deletions(-)
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java
index 9930f9f2707..6c177abbf86 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java
@@ -1000,6 +1000,9 @@ public class IoTDBConfigNodeReceiver extends
IoTDBFileReceiver {
return
ConfigNodeDescriptor.getInstance().getConf().getPipeReceiverFileDir();
}
+ @Override
+ protected void markFileBaseDirStateAbnormal(String dir) {}
+
@Override
protected String getSenderHost() {
final IClientSession session = SESSION_MANAGER.getCurrSession();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileException.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileException.java
index 0d1c4c53882..d681a24d029 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileException.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadFileException.java
@@ -31,4 +31,8 @@ public class LoadFileException extends IoTDBException {
public LoadFileException(Exception exception) {
super(exception, TSStatusCode.LOAD_FILE_ERROR.getStatusCode());
}
+
+ public LoadFileException(String message, Exception exception) {
+ super(message, exception, TSStatusCode.LOAD_FILE_ERROR.getStatusCode());
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTsFileBuilder.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTsFileBuilder.java
index 8b97cafe4c8..873d55d2b7e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTsFileBuilder.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTsFileBuilder.java
@@ -81,27 +81,35 @@ public abstract class PipeTsFileBuilder {
}
}
}
-
- final File baseDir =
- new File(FOLDER_MANAGER.get().getNextFolder(),
Long.toString(currentBatchId.get()));
- if (baseDir.exists()) {
- FileUtils.deleteQuietly(baseDir);
- }
- if (!baseDir.exists() && !baseDir.mkdirs()) {
- LOGGER.warn(
- "Batch id = {}: Failed to create batch file dir {}.",
- currentBatchId.get(),
- baseDir.getPath());
+ synchronized (FOLDER_MANAGER) {
+ File baseDir =
+ FOLDER_MANAGER
+ .get()
+ .getNextWithRetry(
+ folder -> {
+ File dir = new File(folder,
Long.toString(currentBatchId.get()));
+ FileUtils.deleteQuietly(dir);
+ if (dir.mkdirs()) {
+ LOGGER.info(
+ "Batch id = {}: Create batch dir successfully, batch
file dir = {}.",
+ currentBatchId.get(),
+ dir.getPath());
+ return dir;
+ }
+ LOGGER.warn(
+ "Batch id = {}: Failed to create batch file dir {}.",
+ currentBatchId.get(),
+ dir.getPath());
+ return null;
+ });
+ if (baseDir != null) {
+ return baseDir;
+ }
throw new PipeException(
String.format(
"Failed to create batch file dir %s. (Batch id = %s)",
baseDir.getPath(), currentBatchId.get()));
}
- LOGGER.info(
- "Batch id = {}: Create batch dir successfully, batch file dir = {}.",
- currentBatchId.get(),
- baseDir.getPath());
- return baseDir;
}
public abstract void bufferTableModelTablet(String dataBase, Tablet tablet);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java
index 4e5163a5bb0..82c39729dfd 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java
@@ -1002,11 +1002,6 @@ public class PipeConsensusReceiver {
tsFileWriter.getWritingFile().getPath());
}
- private String getReceiverFileBaseDir() throws
DiskSpaceInsufficientException {
- // Get next receiver file base dir by folder manager
- return Objects.isNull(folderManager) ? null :
folderManager.getNextFolder();
- }
-
private void initiateTsFileBufferFolder(List<String> receiverBaseDirsName)
throws IOException {
// initiate receiverFileDirs
for (String receiverFileBaseDir : receiverBaseDirsName) {
@@ -1230,44 +1225,48 @@ public class PipeConsensusReceiver {
}
public void rollToNextWritingPath() throws IOException,
DiskSpaceInsufficientException {
- String receiverBasePath;
- try {
- receiverBasePath = getReceiverFileBaseDir();
- } catch (Exception e) {
- LOGGER.warn(
- "Failed to init pipeConsensus receiver file folder manager because
all disks of folders are full.",
- e);
- throw e;
- }
- if (Objects.isNull(receiverBasePath)) {
- LOGGER.warn(
- "PipeConsensus-PipeName-{}: Failed to get pipeConsensus receiver
file base directory, because folderManager is null. May because the disk is
full.",
- consensusPipeName.toString());
- throw new DiskSpaceInsufficientException(receiveDirs);
+ if (folderManager == null) {
+ throw new IOException(
+ String.format(
+ "PipeConsensus-PipeName-%s: Failed to create tsFileWriter-%d
receiver file dir",
+ consensusPipeName, index));
}
+ this.localWritingDir =
+ folderManager.getNextWithRetry(
+ receiverBasePath -> {
+ if (receiverBasePath == null) {
+ LOGGER.warn(
+ "PipeConsensus-PipeName-{}: Failed to get base
directory", consensusPipeName);
+ return null;
+ }
+ File writingDir = new File(receiverBasePath + File.separator +
index);
+ deleteFileOrDirectoryIfExists(
+ writingDir,
+ String.format(
+ "TsFileWriter-%s roll to new dir and delete last
writing dir", index));
- String localWritingDirPath = receiverBasePath + File.separator + index;
- this.localWritingDir = new File(localWritingDirPath);
- // Remove exists dir
- deleteFileOrDirectoryIfExists(
- this.localWritingDir,
- String.format("TsFileWriter-%s roll to new dir and delete last
writing dir", index));
- if (!this.localWritingDir.mkdirs()) {
- LOGGER.warn(
- "PipeConsensus-PipeName-{}: Failed to create receiver
tsFileWriter-{} file dir {}. May because authority or dir already exists etc.",
- consensusPipeName,
- index,
- this.localWritingDir.getPath());
+ if (writingDir.mkdirs()) {
+ LOGGER.info(
+ "PipeConsensus-PipeName-{}: tsfileWriter-{} roll to
writing path {}",
+ consensusPipeName,
+ index,
+ writingDir.getPath());
+ return writingDir;
+ }
+ LOGGER.warn(
+ "PipeConsensus-PipeName-{}: Failed to create receiver
tsFileWriter-{} file dir {}",
+ consensusPipeName,
+ index,
+ writingDir.getPath());
+ return null;
+ });
+
+ if (this.localWritingDir == null) {
throw new IOException(
String.format(
- "PipeConsensus-PipeName-%s: Failed to create tsFileWriter-%d
receiver file dir %s. May because authority or dir already exists etc.",
- consensusPipeName, index, this.localWritingDir.getPath()));
+ "PipeConsensus-PipeName-%s: Failed to create tsFileWriter-%d
receiver file dir",
+ consensusPipeName, index));
}
- LOGGER.info(
- "PipeConsensus-PipeName-{}: tsfileWriter-{} roll to writing path {}",
- consensusPipeName,
- index,
- localWritingDirPath);
}
public File getLocalWritingDir() {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
index b8634f92b64..720ecfa76ac 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
@@ -508,6 +508,11 @@ public class IoTDBDataNodeReceiver extends
IoTDBFileReceiver {
return Objects.isNull(folderManager) ? null :
folderManager.getNextFolder();
}
+ @Override
+ protected void markFileBaseDirStateAbnormal(String dir) {
+ folderManager.updateFolderState(dir, FolderManager.FolderState.ABNORMAL);
+ }
+
@Override
protected String getSenderHost() {
final IClientSession session = SESSION_MANAGER.getCurrSession();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
index 57ae347b48b..5c5c8e490ce 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
@@ -288,28 +288,57 @@ public class SnapshotLoader {
Map<String, String> fileTarget = new HashMap<>();
for (File file : files) {
String fileKey = file.getName().split("\\.")[0];
- String dataDir;
- if (fileTarget.containsKey(fileKey)) {
- dataDir = fileTarget.get(fileKey);
- } else {
- dataDir = folderManager.getNextFolder();
- fileTarget.put(fileKey, dataDir);
- }
- File targetFile =
- new File(dataDir + File.separator + targetSuffix + File.separator +
file.getName());
- if (!targetFile.getParentFile().exists() &&
!targetFile.getParentFile().mkdirs()) {
+ String dataDir = fileTarget.get(fileKey);
+
+ try {
+ folderManager.getNextWithRetry(
+ currentDataDir -> {
+ String effectiveDir = (dataDir != null) ? dataDir :
currentDataDir;
+ File targetFile =
+ new File(
+ effectiveDir
+ + File.separator
+ + targetSuffix
+ + File.separator
+ + file.getName());
+
+ try {
+ if (!targetFile.getParentFile().exists() &&
!targetFile.getParentFile().mkdirs()) {
+ throw new IOException(
+ String.format(
+ "Cannot create directory %s",
+ targetFile.getParentFile().getAbsolutePath()));
+ }
+
+ try {
+ Files.createLink(targetFile.toPath(), file.toPath());
+ LOGGER.debug("Created hard link from {} to {}", file,
targetFile);
+ return targetFile;
+ } catch (IOException e) {
+ LOGGER.info(
+ "Cannot create link from {} to {}, fallback to copy",
file, targetFile);
+ }
+
+ Files.copy(file.toPath(), targetFile.toPath());
+ fileTarget.put(fileKey, effectiveDir);
+ return targetFile;
+ } catch (Exception e) {
+ LOGGER.warn(
+ "Failed to process file {} in dir {}: {}",
+ file.getName(),
+ effectiveDir,
+ e.getMessage(),
+ e);
+ throw e;
+ }
+ });
+ } catch (Exception e) {
throw new IOException(
String.format(
- "Cannot create directory %s",
targetFile.getParentFile().getAbsolutePath()));
+ "Failed to process file after retries. Source: %s, Target
suffix: %s",
+ file.getAbsolutePath(), targetSuffix),
+ e);
}
- try {
- Files.createLink(targetFile.toPath(), file.toPath());
- continue;
- } catch (IOException e) {
- LOGGER.info("Cannot create link from {} to {}, try to copy it", file,
targetFile);
- }
-
- Files.copy(file.toPath(), targetFile.toPath());
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
index ea408883553..ca0088479e9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
@@ -24,11 +24,14 @@ import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
+import org.apache.iotdb.db.storageengine.rescon.disk.FolderManager;
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.fileSystem.fsFactory.FSFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -43,6 +46,7 @@ import static
org.apache.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
public class TsFileNameGenerator {
private static FSFactory fsFactory = FSFactoryProducer.getFSFactory();
+ private static final Logger LOGGER =
LoggerFactory.getLogger(TsFileNameGenerator.class);
public static String generateNewTsFilePath(
String tsFileDir,
@@ -65,7 +69,7 @@ public class TsFileNameGenerator {
long version,
int innerSpaceCompactionCount,
int crossSpaceCompactionCount)
- throws DiskSpaceInsufficientException {
+ throws DiskSpaceInsufficientException, IOException {
return generateNewTsFilePathWithMkdir(
sequence,
logicalStorageGroup,
@@ -90,33 +94,36 @@ public class TsFileNameGenerator {
int crossSpaceCompactionCount,
int tierLevel,
String customSuffix)
- throws DiskSpaceInsufficientException {
- String tsFileDir =
- generateTsFileDir(
- sequence, logicalStorageGroup, virtualStorageGroup,
timePartitionId, tierLevel);
- fsFactory.getFile(tsFileDir).mkdirs();
- return tsFileDir
- + File.separator
- + generateNewTsFileName(
- time, version, innerSpaceCompactionCount,
crossSpaceCompactionCount, customSuffix);
- }
-
- public static String generateTsFileDir(
- boolean sequence,
- String logicalStorageGroup,
- String virtualStorageGroup,
- long timePartitionId,
- int tierLevel)
- throws DiskSpaceInsufficientException {
- TierManager tierManager = TierManager.getInstance();
- String baseDir = tierManager.getNextFolderForTsFile(tierLevel, sequence);
- return baseDir
- + File.separator
- + logicalStorageGroup
- + File.separator
- + virtualStorageGroup
- + File.separator
- + timePartitionId;
+ throws DiskSpaceInsufficientException, IOException {
+ FolderManager folderManager =
TierManager.getInstance().getFolderManager(tierLevel, sequence);
+ try {
+ return folderManager.getNextWithRetry(
+ baseDir -> {
+ String tsFileDir =
+ baseDir
+ + File.separator
+ + logicalStorageGroup
+ + File.separator
+ + virtualStorageGroup
+ + File.separator
+ + timePartitionId;
+ fsFactory.getFile(tsFileDir).mkdirs();
+ return tsFileDir
+ + File.separator
+ + generateNewTsFileName(
+ time,
+ version,
+ innerSpaceCompactionCount,
+ crossSpaceCompactionCount,
+ customSuffix);
+ });
+ } catch (DiskSpaceInsufficientException e) {
+ LOGGER.error("All disks are full, cannot create tsfile directory", e);
+ throw new IOException("Disk space insufficient", e);
+ } catch (Exception e) {
+ LOGGER.warn("Failed to create tsfile directory after retries", e);
+ throw new IOException("Failed to create directory after retries", e);
+ }
}
public static String generateNewTsFileName(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java
index e98086e5146..119f262ea66 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java
@@ -56,17 +56,18 @@ public abstract class AbstractNodeAllocationStrategy
implements NodeAllocationSt
}
protected IWALNode createWALNode(String identifier) {
- String folder;
- // get wal folder
try {
- folder = folderManager.getNextFolder();
+ return folderManager.getNextWithRetry(
+ folder -> new WALNode(identifier, folder + File.separator +
identifier));
} catch (DiskSpaceInsufficientException e) {
logger.error("Fail to create wal node because all disks of wal folders
are full.", e);
return WALFakeNode.getFailureInstance(e);
+ } catch (Exception e) {
+ logger.warn("Failed to create WAL node after retries for identifier: " +
identifier, e);
+ return WALFakeNode.getFailureInstance(
+ new IOException(
+ "Failed to create WAL node after retries for identifier: " +
identifier, e));
}
- folder = folder + File.separator + identifier;
- // create new wal node
- return createWALNode(identifier, folder);
}
protected IWALNode createWALNode(String identifier, String folder) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index 390dc984696..a0cdebdf030 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -222,12 +222,14 @@ public class LoadTsFileManager {
uuid,
o -> {
try {
- return new TsFileWriterManager(new File(getNextFolder(),
uuid));
+ return getFolderManager()
+ .getNextWithRetry(folder -> new TsFileWriterManager(new
File(folder, uuid)));
} catch (DiskSpaceInsufficientException e) {
exception.set(e);
return null;
}
});
+
if (exception.get() != null || writerManager == null) {
throw new IOException(
"Failed to create TsFileWriterManager for uuid "
@@ -255,7 +257,7 @@ public class LoadTsFileManager {
}
}
- private String getNextFolder() throws DiskSpaceInsufficientException {
+ private FolderManager getFolderManager() throws
DiskSpaceInsufficientException {
if (CONFIG.getLoadTsFileDirs() != LOAD_BASE_DIRS.get()) {
synchronized (FOLDER_MANAGER) {
if (CONFIG.getLoadTsFileDirs() != LOAD_BASE_DIRS.get()) {
@@ -263,7 +265,7 @@ public class LoadTsFileManager {
FOLDER_MANAGER.set(
new FolderManager(
Arrays.asList(LOAD_BASE_DIRS.get()),
DirectoryStrategyType.SEQUENCE_STRATEGY));
- return FOLDER_MANAGER.get().getNextFolder();
+ return FOLDER_MANAGER.get();
}
}
}
@@ -274,12 +276,12 @@ public class LoadTsFileManager {
FOLDER_MANAGER.set(
new FolderManager(
Arrays.asList(LOAD_BASE_DIRS.get()),
DirectoryStrategyType.SEQUENCE_STRATEGY));
- return FOLDER_MANAGER.get().getNextFolder();
+ return FOLDER_MANAGER.get();
}
}
}
- return FOLDER_MANAGER.get().getNextFolder();
+ return FOLDER_MANAGER.get();
}
public boolean loadAll(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/ILoadDiskSelector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/ILoadDiskSelector.java
index b14f0c4b5df..269228e9fe8 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/ILoadDiskSelector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/ILoadDiskSelector.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.storageengine.load.disk;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.exception.load.LoadFileException;
import java.io.File;
@@ -32,7 +33,7 @@ public interface ILoadDiskSelector {
long filePartitionId,
String tsfileName,
int tierLevel)
- throws DiskSpaceInsufficientException;
+ throws DiskSpaceInsufficientException, LoadFileException;
enum LoadDiskSelectorType {
MIN_IO_FIRST("MIN_IO_FIRST"),
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/InheritSystemMultiDisksStrategySelector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/InheritSystemMultiDisksStrategySelector.java
index 16e420c5e2f..7739c42a538 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/InheritSystemMultiDisksStrategySelector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/InheritSystemMultiDisksStrategySelector.java
@@ -20,22 +20,26 @@
package org.apache.iotdb.db.storageengine.load.disk;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.exception.load.LoadFileException;
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.fileSystem.fsFactory.FSFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
public class InheritSystemMultiDisksStrategySelector implements
ILoadDiskSelector {
protected final FSFactory fsFactory = FSFactoryProducer.getFSFactory();
+ private static final Logger logger =
+ LoggerFactory.getLogger(InheritSystemMultiDisksStrategySelector.class);
public InheritSystemMultiDisksStrategySelector() {
// empty body
}
- @Override
public File getTargetFile(
File fileToLoad,
String databaseName,
@@ -43,16 +47,30 @@ public class InheritSystemMultiDisksStrategySelector
implements ILoadDiskSelecto
long filePartitionId,
String tsfileName,
int tierLevel)
- throws DiskSpaceInsufficientException {
- // inherit system multi-disks select strategy, see configuration
`dn_multi_dir_strategy`
- return fsFactory.getFile(
- TierManager.getInstance().getNextFolderForTsFile(tierLevel, false),
- databaseName
- + File.separatorChar
- + dataRegionId
- + File.separatorChar
- + filePartitionId
- + File.separator
- + tsfileName);
+ throws DiskSpaceInsufficientException, LoadFileException {
+ try {
+ return TierManager.getInstance()
+ .getFolderManager(tierLevel, false)
+ .getNextWithRetry(
+ folder -> {
+ return fsFactory.getFile(
+ folder,
+ databaseName
+ + File.separatorChar
+ + dataRegionId
+ + File.separatorChar
+ + filePartitionId
+ + File.separator
+ + tsfileName);
+ });
+ } catch (DiskSpaceInsufficientException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new LoadFileException(
+ String.format(
+ "Storage allocation failed for %s/%s/%s (tier %d)",
+ databaseName, dataRegionId, filePartitionId, tierLevel),
+ e);
+ }
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/MinIOSelector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/MinIOSelector.java
index f6fac586700..d18be675f25 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/MinIOSelector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/disk/MinIOSelector.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.exception.load.LoadFileException;
import org.apache.iotdb.metrics.utils.FileStoreUtils;
import org.slf4j.Logger;
@@ -76,7 +77,7 @@ public class MinIOSelector extends
InheritSystemMultiDisksStrategySelector {
long filePartitionId,
String tsfileName,
int tierLevel)
- throws DiskSpaceInsufficientException {
+ throws DiskSpaceInsufficientException, LoadFileException {
File targetFile;
String fileDirRoot = null;
try {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java
index 0851a698aba..e90292853f1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java
@@ -32,17 +32,35 @@ import
org.apache.iotdb.db.storageengine.rescon.disk.strategy.SequenceStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
public class FolderManager {
private static final Logger logger =
LoggerFactory.getLogger(FolderManager.class);
+ /** Represents the operational states of a data folder. */
+ public enum FolderState {
+ /** Indicates the folder is functioning normally with no issues. */
+ HEALTHY,
+ /** Indicates the folder has operational problems requiring attention. */
+ ABNORMAL
+ }
+
private final List<String> folders;
+
+ /**
+ * Map storing the state of each folder (HEALTHY/ABNORMAL). Key: folder path
as String Value:
+ * corresponding FolderState enum value
+ */
+ private final Map<String, FolderState> foldersStates = new HashMap<>();
+
private final DirectoryStrategy selectStrategy;
public FolderManager(List<String> folders, DirectoryStrategyType type)
throws DiskSpaceInsufficientException {
this.folders = folders;
+ folders.forEach(dir -> foldersStates.put(dir, FolderState.HEALTHY));
switch (type) {
case SEQUENCE_STRATEGY:
this.selectStrategy = new SequenceStrategy();
@@ -61,6 +79,7 @@ public class FolderManager {
}
try {
this.selectStrategy.setFolders(folders);
+ this.selectStrategy.setFoldersStates(foldersStates);
} catch (DiskSpaceInsufficientException e) {
logger.error("All folders are full, change system mode to read-only.",
e);
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
@@ -69,6 +88,11 @@ public class FolderManager {
}
}
+ public void updateFolderState(String folder, FolderState state) {
+ foldersStates.replace(folder, state);
+ selectStrategy.updateFolderState(folder, state);
+ }
+
public String getNextFolder() throws DiskSpaceInsufficientException {
try {
return folders.get(selectStrategy.nextFolderIndex());
@@ -80,6 +104,45 @@ public class FolderManager {
}
}
+ boolean hasHealthyFolder() {
+ return folders.stream()
+ .anyMatch(
+ folder ->
+ foldersStates.getOrDefault(folder, FolderState.ABNORMAL) ==
FolderState.HEALTHY);
+ }
+
+ @FunctionalInterface
+ public interface ThrowingFunction<T, R, E extends Exception> {
+ R apply(T t) throws E;
+ }
+
+ /*
+ * Encapsulates the retry logic for folder operations
+ * @param folderConsumer The operation to perform on the folder (e.g.,
creating TsFileWriterManager)
+ * @return The result of the operation
+ */
+ public <T, E extends Exception> T getNextWithRetry(ThrowingFunction<String,
T, E> folderConsumer)
+ throws DiskSpaceInsufficientException {
+ String folder = null;
+ while (hasHealthyFolder()) {
+ try {
+ folder = folders.get(selectStrategy.nextFolderIndex());
+ } catch (DiskSpaceInsufficientException e) {
+ logger.error("All folders are full, change system mode to read-only.",
e);
+
CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
+
CommonDescriptor.getInstance().getConfig().setStatusReason(NodeStatus.DISK_FULL);
+ throw e;
+ }
+ try {
+ return folderConsumer.apply(folder);
+ } catch (Exception e) {
+ updateFolderState(folder, FolderState.ABNORMAL);
+ logger.warn("Failed to process folder '" + folder);
+ }
+ }
+ throw new DiskSpaceInsufficientException(folders);
+ }
+
public List<String> getFolders() {
return folders;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
index b86f36b1c49..d929f765607 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
@@ -190,6 +190,10 @@ public class TierManager {
: unSeqTiers.get(tierLevel).getNextFolder();
}
+ public FolderManager getFolderManager(int tierLevel, boolean sequence) {
+ return sequence ? seqTiers.get(tierLevel) : unSeqTiers.get(tierLevel);
+ }
+
public List<String> getAllFilesFolders() {
List<String> folders = new ArrayList<>(seqDir2TierLevel.keySet());
folders.addAll(unSeqDir2TierLevel.keySet());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/DirectoryStrategy.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/DirectoryStrategy.java
index 4cf7764183d..bac8d8e47a8 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/DirectoryStrategy.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/DirectoryStrategy.java
@@ -22,12 +22,17 @@ import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.utils.JVMCommonUtils;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
+import org.apache.iotdb.db.storageengine.rescon.disk.FolderManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+
+import static
org.apache.iotdb.db.storageengine.rescon.disk.FolderManager.FolderState.HEALTHY;
/**
* The basic class of all the strategies of multiple directories. If a user
wants to define his own
@@ -63,6 +68,35 @@ public abstract class DirectoryStrategy {
this.folders = folders;
}
+ /**
+ * Map storing the state of each folder (HEALTHY/ABNORMAL). Key: folder path
as String Value:
+ * corresponding FolderState enum value
+ */
+ Map<String, FolderManager.FolderState> foldersStates = new HashMap<>();
+
+ /**
+ * Replaces the entire folder states mapping with a new one.
+ *
+ * @param foldersStates new mapping of folder paths to their states
+ */
+ public void setFoldersStates(Map<String, FolderManager.FolderState>
foldersStates) {
+ this.foldersStates = foldersStates;
+ }
+
+ /**
+ * Updates the state of a specific folder if it exists in the mapping.
+ *
+ * @param folder path of the folder to update
+ * @param state new state to set for the folder
+ */
+ public void updateFolderState(String folder, FolderManager.FolderState
state) {
+ foldersStates.replace(folder, state);
+ }
+
+ public boolean isUnavailableFolder(String dir) {
+ return (foldersStates.getOrDefault(dir, HEALTHY) != HEALTHY);
+ }
+
/**
* Choose a folder to allocate. The user should implement this method to
define his own strategy.
*
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/MaxDiskUsableSpaceFirstStrategy.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/MaxDiskUsableSpaceFirstStrategy.java
index e75c0388d6b..dccedaad207 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/MaxDiskUsableSpaceFirstStrategy.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/MaxDiskUsableSpaceFirstStrategy.java
@@ -35,6 +35,9 @@ public class MaxDiskUsableSpaceFirstStrategy extends
DirectoryStrategy {
for (int i = 0; i < folders.size(); i++) {
String folder = folders.get(i);
+ if (isUnavailableFolder(folder)) {
+ continue;
+ }
if (!JVMCommonUtils.hasSpace(folder)) {
continue;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java
index d02083f03c2..70ceb70573e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/MinFolderOccupiedSpaceFirstStrategy.java
@@ -36,6 +36,9 @@ public class MinFolderOccupiedSpaceFirstStrategy extends
DirectoryStrategy {
for (int i = 0; i < folders.size(); i++) {
String folder = folders.get(i);
+ if (isUnavailableFolder(folder)) {
+ continue;
+ }
if (!JVMCommonUtils.hasSpace(folder)) {
continue;
}
@@ -45,6 +48,7 @@ public class MinFolderOccupiedSpaceFirstStrategy extends
DirectoryStrategy {
space = JVMCommonUtils.getOccupiedSpace(folder);
} catch (IOException e) {
LOGGER.error("Cannot calculate occupied space for path {}.", folder,
e);
+ continue;
}
if (space < minSpace) {
minSpace = space;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/RandomOnDiskUsableSpaceStrategy.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/RandomOnDiskUsableSpaceStrategy.java
index cf4b911c0bb..e564011dc73 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/RandomOnDiskUsableSpaceStrategy.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/RandomOnDiskUsableSpaceStrategy.java
@@ -58,6 +58,10 @@ public class RandomOnDiskUsableSpaceStrategy extends
DirectoryStrategy {
List<Long> spaceList = new ArrayList<>();
for (int i = 0; i < folders.size(); i++) {
String folder = folders.get(i);
+ if (isUnavailableFolder(folder)) {
+ spaceList.add(0L);
+ continue;
+ }
spaceList.add(JVMCommonUtils.getUsableSpace(folder));
}
return spaceList;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/SequenceStrategy.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/SequenceStrategy.java
index ba7265babd5..881623a1882 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/SequenceStrategy.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/strategy/SequenceStrategy.java
@@ -60,7 +60,7 @@ public class SequenceStrategy extends DirectoryStrategy {
private int tryGetNextIndex(int start) throws DiskSpaceInsufficientException
{
int index = (start + 1) % folders.size();
String dir = folders.get(index);
- while (!JVMCommonUtils.hasSpace(dir)) {
+ while (isUnavailableFolder(dir) || !JVMCommonUtils.hasSpace(dir)) {
File dirFile = FSFactoryProducer.getFSFactory().getFile(dir);
Long lastPrintTime = dirLastPrintTimeMap.computeIfAbsent(index, i ->
-1L);
@@ -68,7 +68,7 @@ public class SequenceStrategy extends DirectoryStrategy {
long freeSpace = dirFile.getFreeSpace();
long totalSpace = dirFile.getTotalSpace();
LOGGER.warn(
- "{} is above the warning threshold, free space {}, total space{}",
+ "{} is above the warning threshold, or not accessible, free space
{}, total space {}",
dir,
freeSpace,
totalSpace);
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java
index 69e5664e127..9d84f4d6e0d 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java
@@ -156,48 +156,56 @@ public abstract class IoTDBFileReceiver implements
IoTDBReceiver {
}
}
- final String receiverFileBaseDir;
- try {
- receiverFileBaseDir = getReceiverFileBaseDir();
- if (Objects.isNull(receiverFileBaseDir)) {
+ String receiverFileBaseDir;
+ File newReceiverDir = null;
+ for (int retryTimes = 0; retryTimes <= 1; retryTimes++) {
+ try {
+ receiverFileBaseDir = getReceiverFileBaseDir();
+ if (Objects.isNull(receiverFileBaseDir)) {
+ LOGGER.warn(
+ "Receiver id = {}: Failed to init pipe receiver file folder
manager because all disks of folders are full.",
+ receiverId.get());
+ return new
TPipeTransferResp(StatusUtils.getStatus(TSStatusCode.DISK_SPACE_INSUFFICIENT));
+ }
+ } catch (Exception e) {
LOGGER.warn(
- "Receiver id = {}: Failed to init pipe receiver file folder
manager because all disks of folders are full.",
- receiverId.get());
+ "Receiver id = {}: Failed to create pipe receiver file folder
because all disks of folders are full.",
+ receiverId.get(),
+ e);
return new
TPipeTransferResp(StatusUtils.getStatus(TSStatusCode.DISK_SPACE_INSUFFICIENT));
}
- } catch (Exception e) {
- LOGGER.warn(
- "Receiver id = {}: Failed to create pipe receiver file folder
because all disks of folders are full.",
- receiverId.get(),
- e);
- return new
TPipeTransferResp(StatusUtils.getStatus(TSStatusCode.DISK_SPACE_INSUFFICIENT));
- }
- // Create a new receiver file dir
- final File newReceiverDir = new File(receiverFileBaseDir,
Long.toString(receiverId.get()));
- if (!newReceiverDir.exists() && !newReceiverDir.mkdirs()) {
+ try {
+ // Create a new receiver file dir
+ newReceiverDir = new File(receiverFileBaseDir,
Long.toString(receiverId.get()));
+ if (newReceiverDir.exists() || newReceiverDir.mkdirs()) {
+ receiverFileDirWithIdSuffix.set(newReceiverDir);
+ LOGGER.info(
+ "Receiver id = {}: Handshake successfully! Sender's host = {},
port = {}. Receiver's file dir = {}.",
+ receiverId.get(),
+ getSenderHost(),
+ getSenderPort(),
+ newReceiverDir.getPath());
+ return new TPipeTransferResp(RpcUtils.SUCCESS_STATUS);
+ }
+ } catch (Exception ignored) {
+ }
LOGGER.warn(
"Receiver id = {}: Failed to create receiver file dir {}.",
receiverId.get(),
newReceiverDir.getPath());
- return new TPipeTransferResp(
- RpcUtils.getStatus(
- TSStatusCode.PIPE_HANDSHAKE_ERROR,
- String.format("Failed to create receiver file dir %s.",
newReceiverDir.getPath())));
+ markFileBaseDirStateAbnormal(receiverFileBaseDir);
}
- receiverFileDirWithIdSuffix.set(newReceiverDir);
-
- LOGGER.info(
- "Receiver id = {}: Handshake successfully! Sender's host = {}, port =
{}. Receiver's file dir = {}.",
- receiverId.get(),
- getSenderHost(),
- getSenderPort(),
- newReceiverDir.getPath());
- return new TPipeTransferResp(RpcUtils.SUCCESS_STATUS);
+ return new TPipeTransferResp(
+ RpcUtils.getStatus(
+ TSStatusCode.PIPE_HANDSHAKE_ERROR,
+ String.format("Failed to create receiver file dir %s.",
newReceiverDir.getPath())));
}
protected abstract String getReceiverFileBaseDir() throws Exception;
+ protected abstract void markFileBaseDirStateAbnormal(String dir);
+
protected abstract String getSenderHost();
protected abstract String getSenderPort();
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
index 2df28b9caae..84f9704e4f0 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/JVMCommonUtils.java
@@ -65,32 +65,44 @@ public class JVMCommonUtils {
* @return
*/
public static long getUsableSpace(String dir) {
- File dirFile = FSFactoryProducer.getFSFactory().getFile(dir);
- dirFile.mkdirs();
- return IOUtils.retryNoException(5, 2000L, dirFile::getFreeSpace, space ->
space > 0).orElse(0L);
+ try {
+ File dirFile = FSFactoryProducer.getFSFactory().getFile(dir);
+ dirFile.mkdirs();
+ return IOUtils.retryNoException(5, 2000L, dirFile::getFreeSpace, space
-> space > 0)
+ .orElse(0L);
+ } catch (Exception e) {
+ LOGGER.error("Unexpected error checking disk space for directory: {}",
dir, e);
+ return 0L;
+ }
}
public static double getDiskFreeRatio(String dir) {
- File dirFile = FSFactoryProducer.getFSFactory().getFile(dir);
- if (!dirFile.mkdirs()) {
- // This may solve getFreeSpace() == 0?
- dirFile = new File(dir);
- }
- long freeSpace =
- IOUtils.retryNoException(5, 2000L, dirFile::getFreeSpace, space ->
space > 0).orElse(0L);
- if (freeSpace == 0) {
- LOGGER.warn("Cannot get free space for {} after retries, please check
the disk status", dir);
- }
- long totalSpace = dirFile.getTotalSpace();
- double ratio = 1.0 * freeSpace / totalSpace;
- if (ratio <= diskSpaceWarningThreshold) {
- LOGGER.warn(
- "{} is above the warning threshold, free space {}, total space {}",
- dir,
- freeSpace,
- totalSpace);
+ try {
+ File dirFile = FSFactoryProducer.getFSFactory().getFile(dir);
+ if (!dirFile.mkdirs()) {
+ // This may solve getFreeSpace() == 0?
+ dirFile = new File(dir);
+ }
+ long freeSpace =
+ IOUtils.retryNoException(5, 2000L, dirFile::getFreeSpace, space ->
space > 0).orElse(0L);
+ if (freeSpace == 0) {
+ LOGGER.warn(
+ "Cannot get free space for {} after retries, please check the disk
status", dir);
+ }
+ long totalSpace = dirFile.getTotalSpace();
+ double ratio = 1.0 * freeSpace / totalSpace;
+ if (ratio <= diskSpaceWarningThreshold) {
+ LOGGER.warn(
+ "{} is above the warning threshold, free space {}, total space {}",
+ dir,
+ freeSpace,
+ totalSpace);
+ }
+ return ratio;
+ } catch (Exception e) {
+ LOGGER.error("Unexpected error checking disk space for {}", dir, e);
+ return 0;
}
- return ratio;
}
public static boolean hasSpace(String dir) {
