This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch fix_flush_error_due_to_compressionRatio in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5945a6078353778d702d2411d1a0a692689a3e21 Author: HTHou <[email protected]> AuthorDate: Wed Jul 17 11:08:29 2024 +0800 fix flush failed due to negative compression ratio --- .../dataregion/flush/CompressionRatio.java | 34 ++++++++++++++------ .../dataregion/flush/CompressionRatioTest.java | 37 ++++++++++++++++++++++ 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java index 1e37cb9e8ec..6408c47b044 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java @@ -144,11 +144,18 @@ public class CompressionRatio { int maxRatioIndex = 0; for (int i = 0; i < ratioFiles.length; i++) { String[] fileNameArray = ratioFiles[i].getName().split("-"); - long diskSize = Long.parseLong(fileNameArray[2]); - if (diskSize > totalDiskSize) { - totalMemorySize = new AtomicLong(Long.parseLong(fileNameArray[1])); - totalDiskSize = diskSize; - maxRatioIndex = i; + // fileNameArray.length != 3 means the compression ratio may be negative, ignore it + if (fileNameArray.length == 3) { + try { + long diskSize = Long.parseLong(fileNameArray[2]); + if (diskSize > totalDiskSize) { + totalMemorySize = new AtomicLong(Long.parseLong(fileNameArray[1])); + totalDiskSize = diskSize; + maxRatioIndex = i; + } + } catch (NumberFormatException ignore) { + // ignore illegal compression file name + } } } LOGGER.debug( @@ -165,11 +172,18 @@ public class CompressionRatio { totalDiskSize = 1; for (int i = 0; i < ratioFilesBeforeV121.length; i++) { String[] fileNameArray = ratioFilesBeforeV121[i].getName().split("-"); - double currentCompressRatio = - Double.parseDouble(fileNameArray[1]) / Double.parseDouble(fileNameArray[2]); - if (getRatio() < currentCompressRatio) { - totalMemorySize = new AtomicLong((long) currentCompressRatio); - maxRatioIndex = i; + // fileNameArray.length != 3 means the compression ratio may be negative, ignore it + if (fileNameArray.length == 3) { + try { + double currentCompressRatio = + Double.parseDouble(fileNameArray[1]) / Double.parseDouble(fileNameArray[2]); + if (getRatio() < currentCompressRatio) { + totalMemorySize = new AtomicLong((long) currentCompressRatio); + maxRatioIndex = i; + } + } catch (NumberFormatException ignore) { + // ignore illegal compression file name + } } } deleteRedundantFilesByIndex(ratioFilesBeforeV121, maxRatioIndex); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatioTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatioTest.java index 439d1b81e96..b1da6c32111 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatioTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatioTest.java @@ -104,4 +104,41 @@ public class CompressionRatioTest { // largest diskSize to the memory assertEquals(2, compressionRatio.getRatio(), 0.1); } + + @Test + public void testRestoreIllegal1() throws IOException { + Files.createFile( + new File( + directory, + String.format(Locale.ENGLISH, CompressionRatio.RATIO_FILE_PATH_FORMAT, 10, 50)) + .toPath()); + + Files.createFile( + new File( + directory, + String.format(Locale.ENGLISH, CompressionRatio.RATIO_FILE_PATH_FORMAT, -1000, 100)) + .toPath()); + + compressionRatio.restore(); + + // if multiple files exist in the system due to some exceptions, restore the file with the + // largest diskSize to the memory + assertEquals(0.2, compressionRatio.getRatio(), 0.1); + } + + @Test + public void testRestoreIllegal2() throws IOException { + + Files.createFile( + new File( + directory, + String.format(Locale.ENGLISH, CompressionRatio.RATIO_FILE_PATH_FORMAT, -1000, 100)) + .toPath()); + + compressionRatio.restore(); + + // if compression ratio from file is negative, assume the compression ratio is 0 / 0 = NaN + assertEquals(Double.NaN, compressionRatio.getRatio(), 0.1); + } + }
