This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new b553a6f3670 Pipe/Load: Remove MD5 check for identically named TSFile
to speed up Active Load (#15311)
b553a6f3670 is described below
commit b553a6f36700788dc933dc189d7515f313bffd58
Author: Zhenyu Luo <[email protected]>
AuthorDate: Thu Apr 17 15:00:23 2025 +0800
Pipe/Load: Remove MD5 check for identically named TSFile to speed up Active
Load (#15311)
---
.../org/apache/iotdb/commons/utils/FileUtils.java | 114 ++++++++++++++++-----
1 file changed, 88 insertions(+), 26 deletions(-)
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
index f8de487c365..ea023b1487e 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
@@ -52,6 +52,11 @@ public class FileUtils {
private static final int BUFFER_SIZE = 1024;
+ private static final String RENAME_FILE_MESSAGE =
+ "Renamed file {} to {} because it already exists in the target
directory: {}";
+ private static final String COPY_FILE_MESSAGE =
+ "Copy file {} to {} because it already exists in the target directory:
{}";
+
private FileUtils() {}
public static List<File> listFilesRecursively(File dir, FileFilter
fileFilter) {
@@ -384,46 +389,59 @@ public class FileUtils {
throws IOException {
final String sourceFileName = sourceFile.getName();
final File targetFile = new File(targetDir, sourceFileName);
-
if (targetFile.exists()) {
- if (haveSameMD5(sourceFile, targetFile)) {
- org.apache.commons.io.FileUtils.forceDelete(sourceFile);
- LOGGER.info(
- "Deleted the file {} because it already exists in the target
directory: {}",
- sourceFile.getName(),
- targetDir.getAbsolutePath());
- } else {
- renameWithMD5(sourceFile, targetDir);
- LOGGER.info(
- "Renamed file {} to {} because it already exists in the target
directory: {}",
- sourceFile.getName(),
- targetFile.getName(),
- targetDir.getAbsolutePath());
+ // Check if the file sizes are the same
+ if (sourceFile.length() == targetFile.length()) {
+ moveWithMD5Check(sourceFile, targetDir, targetFile);
+ return;
}
+ moveFileRenameWithSize(sourceFile, targetDir);
} else {
org.apache.commons.io.FileUtils.moveFileToDirectory(sourceFile,
targetDir, true);
}
}
+ private static void moveWithMD5Check(
+ final File sourceFile, final File targetDir, final File targetFile)
throws IOException {
+ if (haveSameMD5(sourceFile, targetFile)) {
+ org.apache.commons.io.FileUtils.forceDelete(sourceFile);
+ LOGGER.info(
+ "Deleted the file {} because it already exists in the target
directory: {}",
+ sourceFile.getName(),
+ targetDir.getAbsolutePath());
+ } else {
+ renameWithMD5(sourceFile, targetDir);
+ }
+ }
+
+ private static void moveFileRenameWithSize(File sourceFile, File targetDir)
throws IOException {
+ final File targetFile = renameWithSize(sourceFile, targetDir);
+ org.apache.commons.io.FileUtils.moveFile(
+ sourceFile, targetFile, StandardCopyOption.REPLACE_EXISTING);
+
+ LOGGER.info(
+ RENAME_FILE_MESSAGE,
+ sourceFile.getName(),
+ targetFile.getName(),
+ targetDir.getAbsolutePath());
+ }
+
public static void copyFileWithMD5Check(final File sourceFile, final File
targetDir)
throws IOException {
final String sourceFileName = sourceFile.getName();
final File targetFile = new File(targetDir, sourceFileName);
if (targetFile.exists()) {
- if (!haveSameMD5(sourceFile, targetFile)) {
- final String renameFile = copyFileRenameWithMD5(sourceFile, targetDir);
- LOGGER.info(
- "Copy file {} to {} because it already exists in the target
directory: {}",
- sourceFile.getName(),
- renameFile,
- targetDir.getAbsolutePath());
+ if (sourceFile.length() == targetFile.length()) {
+ copyWithMD5Check(sourceFile, targetDir, targetFile);
+ return;
}
+ copyFileRenameWithSize(sourceFile, targetDir);
} else {
- if (!(targetDir.exists() || targetDir.mkdirs())) {
- final String log =
- String.format("failed to create target directory: %s",
targetDir.getAbsolutePath());
- LOGGER.warn(log);
- throw new IOException(log);
+ try {
+ Files.createDirectories(targetDir.toPath());
+ } catch (IOException e) {
+ LOGGER.warn("failed to create target directory: {}",
targetDir.getAbsolutePath());
+ throw e;
}
Files.copy(
@@ -434,6 +452,36 @@ public class FileUtils {
}
}
+ private static void copyFileRenameWithSize(final File sourceFile, final File
targetDir)
+ throws IOException {
+ final File targetFile = renameWithSize(sourceFile, targetDir);
+ Files.copy(
+ sourceFile.toPath(),
+ targetFile.toPath(),
+ StandardCopyOption.REPLACE_EXISTING,
+ StandardCopyOption.COPY_ATTRIBUTES);
+
+ LOGGER.info(
+ COPY_FILE_MESSAGE, sourceFile.getName(), targetFile.getName(),
targetDir.getAbsolutePath());
+ }
+
+ private static File renameWithSize(final File sourceFile, final File
targetDir) {
+ final String sourceFileBaseName =
FilenameUtils.getBaseName(sourceFile.getName());
+ final String sourceFileExtension =
FilenameUtils.getExtension(sourceFile.getName());
+
+ // If the file sizes are different, rename the source file by appending
its size and the
+ // current timestamp
+ final String newFileName =
+ String.format(
+ "%s_%s_%s.%s",
+ sourceFileBaseName,
+ sourceFile.length(),
+ System.currentTimeMillis(),
+ sourceFileExtension);
+
+ return new File(targetDir, newFileName);
+ }
+
private static boolean haveSameMD5(final File file1, final File file2) {
try (final InputStream is1 = Files.newInputStream(file1.toPath());
final InputStream is2 = Files.newInputStream(file2.toPath())) {
@@ -455,6 +503,20 @@ public class FileUtils {
org.apache.commons.io.FileUtils.moveFile(
sourceFile, targetFile, StandardCopyOption.REPLACE_EXISTING);
+
+ LOGGER.info(
+ RENAME_FILE_MESSAGE,
+ sourceFile.getName(),
+ targetFile.getName(),
+ targetDir.getAbsolutePath());
+ }
+ }
+
+ private static void copyWithMD5Check(
+ final File sourceFile, final File targetDir, final File targetFile)
throws IOException {
+ if (!haveSameMD5(sourceFile, targetFile)) {
+ final String renameFile = copyFileRenameWithMD5(sourceFile, targetDir);
+ LOGGER.info(COPY_FILE_MESSAGE, sourceFile.getName(), renameFile,
targetDir.getAbsolutePath());
}
}
