This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch diskExceptionCounter-fix in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 298191eeb78282d0b89e99c3441358283111bbf1 Author: shuwenwei <[email protected]> AuthorDate: Wed Sep 2 18:36:48 2026 +0800 Extend suspicious disk exception recording to SyncFailedException and disk-error IOExceptions DataNodeExceptionMetrics currently counts only exact FileSystemException in the cause chain. Real disk failures often surface as SyncFailedException (fsync on old IO) or as plain IOException carrying the errno text from reads/writes/force on already-open channels, so they were missed. - recognize SyncFailedException by type (its message is fixed to "Sync failed" without errno text) - recognize IOException whose message contains well-known disk-error errno text: Input/output error, No space left on device, Read-only file system, Structure needs cleaning, No such device or address, Disk quota exceeded - keep excluding FileSystemException subclasses (NoSuchFileException, AccessDeniedException, ...) which usually indicate logical file-state errors --- .../service/metrics/DataNodeExceptionMetrics.java | 44 +++++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java index 42035ca0c09..8e43753534d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java @@ -27,9 +27,15 @@ import org.apache.iotdb.metrics.type.Counter; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.metrics.utils.MetricType; +import java.io.IOException; +import java.io.SyncFailedException; import java.nio.file.FileSystemException; -/** Records DataNode exceptions whose cause chain contains a {@link FileSystemException}. */ +/** + * Records DataNode exceptions whose cause chain contains a suspicious disk I/O exception, i.e. an + * exact {@link FileSystemException}, a {@link SyncFailedException}, or an {@link IOException} + * carrying a well-known disk-error message. + */ public class DataNodeExceptionMetrics implements IMetricSet { private static final DataNodeExceptionMetrics INSTANCE = new DataNodeExceptionMetrics(); @@ -54,18 +60,44 @@ public class DataNodeExceptionMetrics implements IMetricSet { } public void recordSuspiciousDiskException(Throwable throwable) { - // Uses exact type matching instead of instanceof because subclasses of - // FileSystemException (e.g. NoSuchFileException, NotDirectoryException, - // AccessDeniedException) usually indicate logical file-state errors rather - // than real disk failures, so they should not be counted here. for (Throwable current = throwable; current != null; current = current.getCause()) { - if (current.getClass() == FileSystemException.class) { + if (isSuspiciousDiskException(current)) { suspiciousDiskExceptionCounter.inc(); return; } } } + private static boolean isSuspiciousDiskException(Throwable throwable) { + // 1) exact FileSystemException. Subclasses (e.g. NoSuchFileException, + // NotDirectoryException, AccessDeniedException) usually indicate logical + // file-state errors rather than real disk failures, so they are not counted here. + if (throwable.getClass() == FileSystemException.class) { + return true; + } + // 2) SyncFailedException: fsync/force failure is almost always a real disk problem. + // Its message is fixed to "Sync failed" without the errno text, so it can only be + // recognized by its type. + if (throwable instanceof SyncFailedException) { + return true; + } + // 3) Plain IOException thrown by reads/writes/force on already-open channels carries + // the errno text in its message. Match the well-known disk-error messages instead of + // counting every IOException (EOF, closed stream, etc. are not disk problems). + if (throwable instanceof IOException) { + String message = throwable.getMessage(); + if (message != null) { + return message.contains("Input/output error") + || message.contains("No space left on device") + || message.contains("Read-only file system") + || message.contains("Structure needs cleaning") + || message.contains("No such device or address") + || message.contains("Disk quota exceeded"); + } + } + return false; + } + public static DataNodeExceptionMetrics getInstance() { return INSTANCE; }
