This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 43dfd880740 Pipe: avoid supply event with failed tsfile resource pin &
apply double-checked locking for exists-and-mkdirs operation (#13998) (#14065)
43dfd880740 is described below
commit 43dfd88074048a7af3fa1251cabeb032218e6f8f
Author: V_Galaxy <[email protected]>
AuthorDate: Wed Nov 13 10:40:18 2024 +0800
Pipe: avoid supply event with failed tsfile resource pin & apply
double-checked locking for exists-and-mkdirs operation (#13998) (#14065)
---
.../PipeHistoricalDataRegionTsFileExtractor.java | 6 ++--
.../hardlink/PipeWALHardlinkResourceManager.java | 20 ++---------
.../org/apache/iotdb/commons/utils/FileUtils.java | 40 ++++++++++++++--------
3 files changed, 32 insertions(+), 34 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
index 6c42932f0e7..71f62e98871 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileExtractor.java
@@ -465,15 +465,17 @@ public class PipeHistoricalDataRegionTsFileExtractor
implements PipeHistoricalDa
.collect(Collectors.toList());
resourceList.addAll(unsequenceTsFileResources);
- resourceList.forEach(
+ resourceList.removeIf(
resource -> {
// Pin the resource, in case the file is removed by compaction
or anything.
// Will unpin it after the PipeTsFileInsertionEvent is created
and pinned.
try {
PipeDataNodeResourceManager.tsfile()
.pinTsFileResource(resource, shouldTransferModFile);
+ return false;
} catch (final IOException e) {
- LOGGER.warn("Pipe: failed to pin TsFileResource {}",
resource.getTsFilePath());
+ LOGGER.warn("Pipe: failed to pin TsFileResource {}",
resource.getTsFilePath(), e);
+ return true;
}
});
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
index 7570b83fc73..eebf766dc36 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/hardlink/PipeWALHardlinkResourceManager.java
@@ -21,15 +21,14 @@ package org.apache.iotdb.db.pipe.resource.wal.hardlink;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
+import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.pipe.resource.wal.PipeWALResourceManager;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALEntryHandler;
import java.io.File;
import java.io.IOException;
-import java.nio.file.FileSystems;
import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
@@ -80,7 +79,7 @@ public class PipeWALHardlinkResourceManager extends
PipeWALResourceManager {
// if the file is a wal, and there is no related hardlink in pipe dir,
create a hardlink to pipe
// dir, maintain a reference count for the hardlink, and return the
hardlink.
hardlinkToReferenceMap.put(hardlink.getPath(), 1);
- return createHardlink(file, hardlink);
+ return FileUtils.createHardLink(file, hardlink);
}
private boolean increaseReferenceIfExists(final String path) {
@@ -128,21 +127,6 @@ public class PipeWALHardlinkResourceManager extends
PipeWALResourceManager {
return builder.toString();
}
- private static File createHardlink(final File sourceFile, final File
hardlink)
- throws IOException {
- if (!hardlink.getParentFile().exists() &&
!hardlink.getParentFile().mkdirs()) {
- throw new IOException(
- String.format(
- "failed to create hardlink %s for file %s: failed to create
parent dir %s",
- hardlink.getPath(), sourceFile.getPath(),
hardlink.getParentFile().getPath()));
- }
-
- final Path sourcePath =
FileSystems.getDefault().getPath(sourceFile.getAbsolutePath());
- final Path linkPath =
FileSystems.getDefault().getPath(hardlink.getAbsolutePath());
- Files.createLink(linkPath, sourcePath);
- return hardlink;
- }
-
/**
* given a hardlink, decrease its reference count, if the reference count is
0, delete the file.
* if the given file is not a hardlink, do nothing.
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
index 420dfda6c1f..2ae4858ec7e 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
@@ -105,12 +105,14 @@ public class FileUtils {
sourceDir.getAbsolutePath());
return false;
}
- if (!targetDir.exists()) {
- if (!targetDir.mkdirs()) {
- LOGGER.error(
- "Failed to copy folder, because failed to create target
folder[{}].",
- targetDir.getAbsolutePath());
- return false;
+ if (!targetDir.exists() && !targetDir.mkdirs()) {
+ synchronized (FileUtils.class) {
+ if (!targetDir.exists() && !targetDir.mkdirs()) {
+ LOGGER.error(
+ "Failed to copy folder, because failed to create target
folder[{}].",
+ targetDir.getAbsolutePath());
+ return false;
+ }
}
} else if (!targetDir.isDirectory()) {
LOGGER.error(
@@ -280,10 +282,14 @@ public class FileUtils {
public static File createHardLink(File sourceFile, File hardlink) throws
IOException {
if (!hardlink.getParentFile().exists() &&
!hardlink.getParentFile().mkdirs()) {
- throw new IOException(
- String.format(
- "failed to create hardlink %s for file %s: failed to create
parent dir %s",
- hardlink.getPath(), sourceFile.getPath(),
hardlink.getParentFile().getPath()));
+ synchronized (FileUtils.class) {
+ if (!hardlink.getParentFile().exists() &&
!hardlink.getParentFile().mkdirs()) {
+ throw new IOException(
+ String.format(
+ "failed to create hardlink %s for file %s: failed to create
parent dir %s",
+ hardlink.getPath(), sourceFile.getPath(),
hardlink.getParentFile().getPath()));
+ }
+ }
}
final Path sourcePath =
FileSystems.getDefault().getPath(sourceFile.getAbsolutePath());
@@ -294,10 +300,16 @@ public class FileUtils {
public static File copyFile(File sourceFile, File targetFile) throws
IOException {
if (!targetFile.getParentFile().exists() &&
!targetFile.getParentFile().mkdirs()) {
- throw new IOException(
- String.format(
- "failed to copy file %s to %s: failed to create parent dir %s",
- sourceFile.getPath(), targetFile.getPath(),
targetFile.getParentFile().getPath()));
+ synchronized (FileUtils.class) {
+ if (!targetFile.getParentFile().exists() &&
!targetFile.getParentFile().mkdirs()) {
+ throw new IOException(
+ String.format(
+ "failed to copy file %s to %s: failed to create parent dir
%s",
+ sourceFile.getPath(),
+ targetFile.getPath(),
+ targetFile.getParentFile().getPath()));
+ }
+ }
}
Files.copy(sourceFile.toPath(), targetFile.toPath());
