This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch force_ci/object_type in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e72ee9a247a3e72b8baf3cff579bfc4829017997 Author: Zhenyu Luo <[email protected]> AuthorDate: Thu Nov 20 19:02:45 2025 +0800 Load: Fixed parameter passing error in MoveFile function (#16748) * Load: Fixed parameter passing error in MoveFile function * update * add ut * update (cherry picked from commit b3509d6d6bf16203137bca4cb4b204e7cbd14654) --- .../org/apache/iotdb/commons/utils/FileUtils.java | 92 +++++++++++++--------- .../apache/iotdb/commons/utils/FileUtilsTest.java | 79 +++++++++++++++++++ 2 files changed, 132 insertions(+), 39 deletions(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java index 5775eb65853..baed8bcd537 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java @@ -418,7 +418,7 @@ public class FileUtils { final String sourceFileName = sourceFile.getName(); final File targetFile = new File(targetDir, sourceFileName); if (targetFile.exists()) { - moveFile(sourceFile, targetFile); + moveFile(sourceFile, targetDir); } else { org.apache.tsfile.external.commons.io.FileUtils.moveFileToDirectory( sourceFile, targetDir, true); @@ -428,31 +428,37 @@ public class FileUtils { private static void moveFile(File sourceFile, File targetDir) throws IOException { String sourceFileName = sourceFile.getName(); final File exitsFile = new File(targetDir, sourceFileName); - try (final FileInputStream is1 = new FileInputStream(sourceFile); - final FileInputStream is2 = new FileInputStream(exitsFile); ) { - long sourceFileSize = is1.getChannel().size(); - long exitsFileSize = is2.getChannel().size(); - if (sourceFileSize != exitsFileSize) { - File file = renameWithSize(sourceFile, sourceFileSize, targetDir); - if (!file.exists()) { - moveFileRename(sourceFile, file); - return; - } - } - String sourceFileMD5 = DigestUtils.md5Hex(is1); - String exitsFileMD5 = DigestUtils.md5Hex(is2); + // First check file sizes + long sourceFileSize = sourceFile.length(); + long existsFileSize = exitsFile.length(); - if (sourceFileMD5.equals(exitsFileMD5)) { - org.apache.tsfile.external.commons.io.FileUtils.forceDelete(sourceFile); - LOGGER.info( - "Deleted the file {} because it already exists in the target directory: {}", - sourceFile.getName(), - targetDir.getAbsolutePath()); - } else { - File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir); + if (sourceFileSize != existsFileSize) { + File file = renameWithSize(sourceFile, sourceFileSize, targetDir); + if (!file.exists()) { moveFileRename(sourceFile, file); } + return; + } + + // If sizes are equal, check MD5 + String sourceFileMD5; + String existsFileMD5; + try (final FileInputStream is1 = new FileInputStream(sourceFile); + final FileInputStream is2 = new FileInputStream(exitsFile); ) { + sourceFileMD5 = DigestUtils.md5Hex(is1); + existsFileMD5 = DigestUtils.md5Hex(is2); + } + + if (sourceFileMD5.equals(existsFileMD5)) { + org.apache.tsfile.external.commons.io.FileUtils.forceDelete(sourceFile); + LOGGER.info( + "Deleted the file {} because it already exists in the target directory: {}", + sourceFile.getName(), + targetDir.getAbsolutePath()); + } else { + File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir); + moveFileRename(sourceFile, file); } } @@ -492,27 +498,35 @@ public class FileUtils { throws IOException { String sourceFileName = sourceFile.getName(); final File exitsFile = new File(targetDir, sourceFileName); - try (final FileInputStream is1 = new FileInputStream(sourceFile); - final FileInputStream is2 = new FileInputStream(exitsFile); ) { - long sourceFileSize = is1.getChannel().size(); - long exitsFileSize = is2.getChannel().size(); - if (sourceFileSize != exitsFileSize) { - File file = renameWithSize(sourceFile, sourceFileSize, targetDir); - if (!file.exists()) { - copyFileRename(sourceFile, file); - return; - } - } - String sourceFileMD5 = DigestUtils.md5Hex(is1); - String exitsFileMD5 = DigestUtils.md5Hex(is2); - if (sourceFileMD5.equals(exitsFileMD5)) { - return; - } - File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir); + // First check file sizes + long sourceFileSize = sourceFile.length(); + long exitsFileSize = exitsFile.length(); + + if (sourceFileSize != exitsFileSize) { + File file = renameWithSize(sourceFile, sourceFileSize, targetDir); if (!file.exists()) { copyFileRename(sourceFile, file); } + return; + } + + // If sizes are equal, check MD5 + String sourceFileMD5; + String exitsFileMD5; + try (final FileInputStream is1 = new FileInputStream(sourceFile); + final FileInputStream is2 = new FileInputStream(exitsFile); ) { + sourceFileMD5 = DigestUtils.md5Hex(is1); + exitsFileMD5 = DigestUtils.md5Hex(is2); + } + + if (sourceFileMD5.equals(exitsFileMD5)) { + return; + } + + File file = renameWithMD5(sourceFile, sourceFileMD5, targetDir); + if (!file.exists()) { + copyFileRename(sourceFile, file); } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/FileUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/FileUtilsTest.java new file mode 100644 index 00000000000..e25755f4c17 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/FileUtilsTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.utils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Collections; + +public class FileUtilsTest { + private File tmpDir; + private File targetDir; + + @Before + public void setUp() throws Exception { + tmpDir = new File(Files.createTempDirectory("load").toUri()); + targetDir = new File(Files.createTempDirectory("target").toUri()); + } + + @After + public void tearDown() throws Exception { + tmpDir.delete(); + targetDir.delete(); + } + + @Test + public void testFileUtils() throws WriteProcessException, IOException { + File tstFile = new File(tmpDir, "1-1-0-0.tsfile"); + File tstFile2 = new File(tmpDir, "2-1-0-0.tsfile"); + generateFile(tstFile); + FileUtils.copyFile(tstFile, tstFile2); + FileUtils.moveFileWithMD5Check(tstFile, targetDir); + tstFile2.renameTo(tstFile); + FileUtils.moveFileWithMD5Check(tstFile, targetDir); + } + + private void generateFile(File tsfile) throws WriteProcessException, IOException { + try (TsFileWriter writer = new TsFileWriter(tsfile)) { + writer.registerAlignedTimeseries( + "root.test.d1", + Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN))); + Tablet tablet = + new Tablet( + "root.test.d1", + Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN))); + for (int i = 0; i < 5; i++) { + tablet.addTimestamp(i, i); + tablet.addValue(i, 0, true); + } + writer.writeTree(tablet); + } + } +}
