This is an automated email from the ASF dual-hosted git repository.

jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new a6bda7729ce [Pipe] Fix the ownership, cleanup and copy fallback of the 
pipe hardlink dir (#19258)
a6bda7729ce is described below

commit a6bda7729ce905cdecfd9b65a646109ba6c5e700
Author: Zhenyu Luo <[email protected]>
AuthorDate: Tue Nov 17 16:41:30 2026 +0800

    [Pipe] Fix the ownership, cleanup and copy fallback of the pipe hardlink 
dir (#19258)
    
    * [Pipe] Fix the ownership, cleanup and copy fallback of the pipe hardlink 
dir
    
    The TsFiles of one DataRegion may be spread over the disks listed by
    dn_data_dirs. The pipe directory of a TsFile is
    <dataDir>/pipe/tsfile[/<pipeName>] of the data directory that contains the
    TsFile, which keeps the hardlink on the same disk by construction. Fix the
    ownership, cleanup and fallback bugs around that layout:
    
    - Give every pipe its own pipe file when the assigner hands over the public
      pipe file, so that the reference count of the public file is not lost and 
a
      pipe never cleans up a directory shared with other pipes.
    - Track every pipe directory of a pipe, so that dropping a pipe removes its
      files from every disk.
    - Never copy a TsFile which is not closed yet, because the copy would be
      truncated: the pin fails and is retried after the close instead.
    - Copy atomically through a temporary file, for every copy and not only for
      TsFiles, so that a crash or a full disk can not leave a truncated file 
under
      a valid pipe file name; a stale target or temporary file is replaced 
instead
      of failing every later pin.
    - Replace a pipe file left by a previous process instead of linking it onto
      itself, which would delete it as a failed hardlink target.
    - Rebuild the directory mapping when dn_data_dirs is hot reloaded, use
      canonical paths, and fall back to the path-derived layout for files 
outside
      the data directories instead of throwing a NullPointerException.
    - Fail the path resolution when the canonical path of a file can not be
      resolved, instead of silently falling back to its absolute path, which 
would
      let a pin succeed with a non-canonical path.
    - Fail the pin when a pipe file of the requesting scope does not exist, 
instead
      of registering a reference count for a missing file.
    - Fix the hardlink-failure warning, which used %s with SLF4J, and only match
      the stale directories created by the startup cleaner.
    
    * [Pipe] Move hardlink and atomic copy helpers to FileUtils
    
    These helpers run on the production pin path, so they belong in FileUtils 
rather than a test-only method on the pipe resource manager.
    
    Co-authored-by: Cursor <[email protected]>
    
    ---------
    
    Co-authored-by: Cursor <[email protected]>
---
 .../apache/iotdb/db/i18n/DataNodePipeMessages.java |  12 ++
 .../apache/iotdb/db/i18n/DataNodePipeMessages.java |  11 +
 .../agent/runtime/PipeDataNodeRuntimeAgent.java    |   4 +
 ...aNodeHardlinkOrCopiedFileDirStartupCleaner.java |  82 ++++---
 .../resource/tsfile/PipeTsFileDirPathMapping.java  | 206 ++++++++++++++++++
 .../resource/tsfile/PipeTsFileDirPathResolver.java | 122 +++++++++++
 .../resource/tsfile/PipeTsFileResourceManager.java | 170 +++++++--------
 .../resource/PipeTsFileResourceManagerTest.java    | 236 +++++++++++++++++++++
 .../tsfile/PipeTsFileDirPathMappingTest.java       | 219 +++++++++++++++++++
 .../apache/iotdb/commons/i18n/UtilMessages.java    |   3 +
 .../apache/iotdb/commons/i18n/UtilMessages.java    |   3 +
 .../org/apache/iotdb/commons/utils/FileUtils.java  |  64 ++++++
 12 files changed, 1009 insertions(+), 123 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index 85f40d07c9a..0dccb0fb044 100644
--- 
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -3168,4 +3168,16 @@ public final class DataNodePipeMessages {
       
EXCEPTION_HISTORICAL_SUMMARY_EXTRACTEDTSFILECOUNT_ARG_SKIPPEDTSFILECOUNT_ARG_SPLITTSFILECOUNT_ARG_UNSPLITTSFILECOUNT_ARG_DELETIONCOUNT_ARG_C8BF3247
 =
           "historical summary: extractedTsFileCount=%s, skippedTsFileCount=%s, 
splitTsFileCount=%s,"
               + " unsplitTsFileCount=%s, deletionCount=%s";
+
+  public static final String
+      
EXCEPTION_FAILED_TO_PIN_THE_PIPE_FILE_ARG_BECAUSE_IT_DOES_NOT_EXIST_1741B8E4 =
+          "Failed to pin the pipe file %s because it does not exist";
+
+  public static final String 
LOG_PIPE_HARDLINK_DIRECTORIES_ARE_RESOLVED_AS_ARG_1DAB3D0F =
+      "Pipe hardlink directories are resolved as {}.";
+
+  public static final String
+      
EXCEPTION_FAILED_TO_PIN_THE_TSFILE_ARG_BY_CREATING_A_HARDLINK_AND_THE_TSFILE_CAN_NOT_BE_COPIED_BECAUSE_IT_IS_NOT_CLOSED_YET_4E346D68
 =
+          "Failed to pin the TsFile %s by creating a hardlink, and the TsFile 
can not be copied"
+              + " because it is not closed yet";
 }
diff --git 
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index 25ec14c341a..579172c77d0 100644
--- 
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2943,4 +2943,15 @@ public final class DataNodePipeMessages {
   public static final String
       
EXCEPTION_HISTORICAL_SUMMARY_EXTRACTEDTSFILECOUNT_ARG_SKIPPEDTSFILECOUNT_ARG_SPLITTSFILECOUNT_ARG_UNSPLITTSFILECOUNT_ARG_DELETIONCOUNT_ARG_C8BF3247
 =
           "历史数据概览:提取的 TsFile 数=%s,跳过的 TsFile 数=%s,拆分的 TsFile 数=%s,未拆分的 TsFile 
数=%s,删除数=%s";
+
+  public static final String
+      
EXCEPTION_FAILED_TO_PIN_THE_PIPE_FILE_ARG_BECAUSE_IT_DOES_NOT_EXIST_1741B8E4 =
+          "pipe 文件 %s 不存在,无法固化";
+
+  public static final String 
LOG_PIPE_HARDLINK_DIRECTORIES_ARE_RESOLVED_AS_ARG_1DAB3D0F =
+      "Pipe 硬链接目录解析结果为 {}。";
+
+  public static final String
+      
EXCEPTION_FAILED_TO_PIN_THE_TSFILE_ARG_BY_CREATING_A_HARDLINK_AND_THE_TSFILE_CAN_NOT_BE_COPIED_BECAUSE_IT_IS_NOT_CLOSED_YET_4E346D68
 =
+          "通过创建硬链接固化 TsFile %s 失败,且该 TsFile 尚未关闭,无法改为拷贝";
 }
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
index 8ca2ee1f52f..87fe67e75b7 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java
@@ -46,6 +46,7 @@ import 
org.apache.iotdb.db.pipe.resource.PipeDataNodeHardlinkOrCopiedFileDirStar
 import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
 import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock;
 import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlockCategory;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileDirPathResolver;
 import org.apache.iotdb.db.pipe.source.schemaregion.SchemaRegionListeningQueue;
 import 
org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
@@ -85,6 +86,9 @@ public class PipeDataNodeRuntimeAgent implements IService {
 
   public synchronized void preparePipeResources(
       final ResourcesInformationHolder resourcesInformationHolder) throws 
StartupException {
+    // Resolve the pipe hardlink directories before any pipe task starts.
+    PipeTsFileDirPathResolver.getInstance();
+
     // Clean sender (connector) hardlink file dir and snapshot dir
     
PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.clean(this::registerPeriodicalJob);
 
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java
index 8dfc4c1a8e6..f93c3577321 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java
@@ -27,6 +27,8 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
+
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.FileVisitResult;
@@ -35,7 +37,10 @@ import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
@@ -64,29 +69,44 @@ public class 
PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner {
 
   private static void cleanTsFileDir(final PeriodicalJobRegistrar 
periodicalJobRegistrar) {
     final String pipeHardlinkBaseDirName = 
PipeConfig.getInstance().getPipeHardlinkBaseDirName();
+    final String pipeHardlinkTsFileDirName =
+        PipeConfig.getInstance().getPipeHardlinkTsFileDirName();
     final List<File> stalePipeDirs = new ArrayList<>();
-    for (final String dataDir : 
IoTDBDescriptor.getInstance().getConfig().getDataDirs()) {
-      final File localDataDir = new File(dataDir);
-      collectInterruptedStalePipeDirs(localDataDir, pipeHardlinkBaseDirName, 
stalePipeDirs);
-
-      final File pipeHardLinkDir = new File(localDataDir, 
pipeHardlinkBaseDirName);
-      if (pipeHardLinkDir.isDirectory()) {
-        moveAsideAndCollect(pipeHardLinkDir, pipeHardlinkBaseDirName, 
stalePipeDirs);
+    for (final File pipeBaseDir : getPipeBaseDirs(pipeHardlinkBaseDirName)) {
+      // Stale pipe dirs left by an older version, e.g., 
<dataDir>/pipe.startup-cleaning-...
+      collectInterruptedStalePipeDirs(
+          pipeBaseDir.getParentFile(), pipeHardlinkBaseDirName, stalePipeDirs);
+      // Stale tsfile dirs under the pipe base dir, e.g., 
<pipeDir>/tsfile.startup-cleaning-...
+      collectInterruptedStalePipeDirs(pipeBaseDir, pipeHardlinkTsFileDirName, 
stalePipeDirs);
+
+      final File pipeTsFileDir = new File(pipeBaseDir, 
pipeHardlinkTsFileDirName);
+      if (pipeTsFileDir.isDirectory()) {
+        moveAsideAndCollect(pipeTsFileDir, stalePipeDirs);
       }
     }
     PERIODICAL_STALE_PIPE_DIR_CLEANER.addStalePipeDirs(stalePipeDirs);
     registerPeriodicalCleanupJob(periodicalJobRegistrar);
   }
 
+  /** Returns the pipe base directory of every local data dir, i.e. one per 
disk. */
+  private static Set<File> getPipeBaseDirs(final String 
pipeHardlinkBaseDirName) {
+    final Set<File> pipeBaseDirs = new LinkedHashSet<>();
+    for (final String dataDir : 
IoTDBDescriptor.getInstance().getConfig().getLocalDataDirs()) {
+      pipeBaseDirs.add(new File(new File(dataDir), pipeHardlinkBaseDirName));
+    }
+    return pipeBaseDirs;
+  }
+
   private static void collectInterruptedStalePipeDirs(
-      final File localDataDir,
-      final String pipeHardlinkBaseDirName,
-      final List<File> stalePipeDirs) {
+      final @Nullable File dir, final String movedDirName, final List<File> 
stalePipeDirs) {
+    if (Objects.isNull(dir)) {
+      return;
+    }
+    // Only the directories this cleaner has moved aside are matched, not any 
similar name.
+    final String stalePipeDirNamePrefix = movedDirName + STALE_PIPE_DIR_SUFFIX;
     final File[] stalePipeDirFiles =
-        localDataDir.listFiles(
-            file ->
-                file.isDirectory()
-                    && file.getName().contains(pipeHardlinkBaseDirName + 
STALE_PIPE_DIR_SUFFIX));
+        dir.listFiles(
+            file -> file.isDirectory() && 
file.getName().startsWith(stalePipeDirNamePrefix));
     if (stalePipeDirFiles == null) {
       return;
     }
@@ -99,43 +119,37 @@ public class 
PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner {
     }
   }
 
-  private static void moveAsideAndCollect(
-      final File pipeHardLinkDir,
-      final String pipeHardlinkBaseDirName,
-      final List<File> stalePipeDirs) {
+  private static void moveAsideAndCollect(final File pipeDir, final List<File> 
stalePipeDirs) {
     try {
-      final File stalePipeDir = moveAside(pipeHardLinkDir, 
pipeHardlinkBaseDirName);
+      final File stalePipeDir = moveAside(pipeDir);
       LOGGER.info(
           
DataNodePipeMessages.PIPE_HARDLINK_DIR_FOUND_MOVED_TO_PERIODICAL_DELETE,
-          pipeHardLinkDir,
+          pipeDir,
           stalePipeDir);
       stalePipeDirs.add(stalePipeDir);
     } catch (final IOException e) {
-      LOGGER.warn(
-          DataNodePipeMessages.PIPE_HARDLINK_DIR_MOVE_FAILED_DELETING_SYNC, 
pipeHardLinkDir, e);
-      
org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(pipeHardLinkDir, 
true);
+      
LOGGER.warn(DataNodePipeMessages.PIPE_HARDLINK_DIR_MOVE_FAILED_DELETING_SYNC, 
pipeDir, e);
+      org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(pipeDir, 
true);
       LOGGER.info(
           DataNodePipeMessages.PIPE_HARDLINK_DIR_FOUND_DELETING_IT_RESULT,
-          pipeHardLinkDir,
-          !pipeHardLinkDir.exists());
+          pipeDir,
+          !pipeDir.exists());
     }
   }
 
-  private static File moveAside(final File pipeHardLinkDir, final String 
pipeHardlinkBaseDirName)
-      throws IOException {
-    final File parentDir = pipeHardLinkDir.getParentFile();
-    if (parentDir == null) {
+  private static File moveAside(final File pipeDir) throws IOException {
+    final File parentDir = pipeDir.getParentFile();
+    if (Objects.isNull(parentDir)) {
       throw new IOException(
-          DataNodePipeMessages.EXCEPTION_FAILED_TO_GET_PARENT_DIR_OF_8CE21C1D 
+ pipeHardLinkDir);
+          DataNodePipeMessages.EXCEPTION_FAILED_TO_GET_PARENT_DIR_OF_8CE21C1D 
+ pipeDir);
     }
 
     final long timestamp = System.currentTimeMillis();
     for (int i = 0; ; ++i) {
       final File stalePipeDir =
-          new File(
-              parentDir, pipeHardlinkBaseDirName + STALE_PIPE_DIR_SUFFIX + 
timestamp + "-" + i);
+          new File(parentDir, pipeDir.getName() + STALE_PIPE_DIR_SUFFIX + 
timestamp + "-" + i);
       if (!stalePipeDir.exists()) {
-        Files.move(pipeHardLinkDir.toPath(), stalePipeDir.toPath());
+        Files.move(pipeDir.toPath(), stalePipeDir.toPath());
         return stalePipeDir;
       }
     }
@@ -161,7 +175,7 @@ public class 
PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner {
     final File stalePipeDirToDelete;
     if (stalePipeDir.isDirectory()) {
       try {
-        stalePipeDirToDelete = moveAside(stalePipeDir, stalePipeDir.getName());
+        stalePipeDirToDelete = moveAside(stalePipeDir);
         LOGGER.info(
             
DataNodePipeMessages.PIPE_HARDLINK_DIR_FOUND_MOVED_TO_PERIODICAL_DELETE,
             stalePipeDir,
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileDirPathMapping.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileDirPathMapping.java
new file mode 100644
index 00000000000..cc47135b890
--- /dev/null
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileDirPathMapping.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.resource.tsfile;
+
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Maps a TsFile, a .resource file or a .mod file to the pipe directory that 
holds its hardlink or
+ * copy, i.e. {@code <dataDir>/<pipe hardlink base dir>/<pipe hardlink tsfile 
dir>[/<pipeName>]} of
+ * the data directory that contains the file.
+ *
+ * <p>A hardlink can only be created on the same file system, and the TsFiles 
of one DataRegion may
+ * be spread over multiple disks, so the directory is resolved per file.
+ */
+class PipeTsFileDirPathMapping {
+
+  private static final Logger LOGGER = 
LoggerFactory.getLogger(PipeTsFileDirPathMapping.class);
+
+  /** Canonical path of a local data directory -> canonical path of its pipe 
base directory. */
+  private final Map<String, String> dataDirPathToPipeBaseDirPath = new 
LinkedHashMap<>();
+
+  private final String pipeHardlinkBaseDirName;
+  private final String pipeHardlinkTsFileDirName;
+
+  PipeTsFileDirPathMapping(
+      final @Nonnull String[] localDataDirs,
+      final @Nonnull String pipeHardlinkBaseDirName,
+      final @Nonnull String pipeHardlinkTsFileDirName) {
+    this.pipeHardlinkBaseDirName = pipeHardlinkBaseDirName;
+    this.pipeHardlinkTsFileDirName = pipeHardlinkTsFileDirName;
+
+    for (final String dataDir : localDataDirs) {
+      final String canonicalDataDir = canonicalDataDirPath(dataDir);
+      dataDirPathToPipeBaseDirPath.put(
+          canonicalDataDir, canonicalDataDir + File.separator + 
pipeHardlinkBaseDirName);
+    }
+    LOGGER.info(
+        
DataNodePipeMessages.LOG_PIPE_HARDLINK_DIRECTORIES_ARE_RESOLVED_AS_ARG_1DAB3D0F,
+        dataDirPathToPipeBaseDirPath);
+  }
+
+  /**
+   * Returns the hardlink or the copied file in the pipe directory of {@code 
pipeName}, or in the
+   * public pipe directory if {@code pipeName} is {@code null}.
+   */
+  File getHardlinkOrCopiedFile(final File file, final @Nullable String 
pipeName)
+      throws IOException {
+    final File pipeTsFileDir = getPipeTsFileDir(file, pipeName);
+    final File parent = file.getAbsoluteFile().getParentFile();
+    if (Objects.nonNull(parent) && 
canonicalPath(parent).equals(pipeTsFileDir.getPath())) {
+      // The file is already the pipe file of this scope, e.g. one left by a 
previous process. It is
+      // reused, and must not be linked or copied onto itself.
+      return file;
+    }
+    return new File(pipeTsFileDir, getRelativeFilePath(file));
+  }
+
+  private File getPipeTsFileDir(final File file, final @Nullable String 
pipeName)
+      throws IOException {
+    // Canonical, so that a symlinked data dir yields one path per physical 
file.
+    final File tsFileDir =
+        new File(getPipeBaseDir(file).getCanonicalFile(), 
pipeHardlinkTsFileDirName);
+    return Objects.isNull(pipeName) ? tsFileDir : new File(tsFileDir, 
pipeName);
+  }
+
+  /** Returns the pipe base directory of the file, which is always on the disk 
of the file. */
+  private File getPipeBaseDir(final File file) throws IOException {
+    // Canonical, so that a symlinked data dir yields one path per physical 
file. A path which can
+    // not be resolved must fail the resolution: a pin on a non-canonical path 
would break the "one
+    // path per physical file" invariant of the reference count map.
+    final String matchedDataDir = findLongestDataDir(canonicalPath(file));
+    if (Objects.nonNull(matchedDataDir)) {
+      return new File(matchedDataDir, pipeHardlinkBaseDirName);
+    }
+
+    final File existingPipeBaseDir = getExistingPipeBaseDir(file);
+    if (Objects.nonNull(existingPipeBaseDir)) {
+      return existingPipeBaseDir;
+    }
+
+    // The file is not below a local data directory, e.g. it has been 
unloaded, so the pipe
+    // directory goes next to the data directory layout the file still belongs 
to.
+    for (File dir = file.getAbsoluteFile().getParentFile();
+        Objects.nonNull(dir);
+        dir = dir.getParentFile()) {
+      if (isDataDirAnchor(dir) && Objects.nonNull(dir.getParentFile())) {
+        return new File(dir.getParentFile(), pipeHardlinkBaseDirName);
+      }
+    }
+    throw failedToGetPipeDir(file, null);
+  }
+
+  /** Returns the pipe base directory the file is already in, or {@code null}. 
*/
+  private @Nullable File getExistingPipeBaseDir(final @Nonnull File file) {
+    File dir = file.getAbsoluteFile().getParentFile();
+    // <pipe base dir>/<pipe hardlink tsfile dir>[/<pipeName>]/<file>
+    for (int depth = 0; depth < 2 && Objects.nonNull(dir); ++depth, dir = 
dir.getParentFile()) {
+      final File baseDir = dir.getParentFile();
+      if (dir.getName().equals(pipeHardlinkTsFileDirName)
+          && Objects.nonNull(baseDir)
+          && baseDir.getName().equals(pipeHardlinkBaseDirName)) {
+        return baseDir;
+      }
+    }
+    return null;
+  }
+
+  /** Returns the name of the pipe file, which is unique among the files of 
one pipe. */
+  private String getRelativeFilePath(final @Nonnull File file) {
+    if (Objects.nonNull(getExistingPipeBaseDir(file))) {
+      // The pipe file directories are flat.
+      return file.getName();
+    }
+
+    final StringBuilder builder = new StringBuilder(file.getName());
+    for (File currentFile = file;
+        !isDataDirAnchor(currentFile) && 
Objects.nonNull(currentFile.getParentFile());
+        currentFile = currentFile.getParentFile()) {
+      builder.insert(0, currentFile.getParentFile().getName() + 
IoTDBConstant.FILE_NAME_SEPARATOR);
+    }
+    return builder.toString();
+  }
+
+  private boolean isDataDirAnchor(final @Nonnull File file) {
+    final String name = file.getName();
+    return name.equals(IoTDBConstant.SEQUENCE_FOLDER_NAME)
+        || name.equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)
+        || name.equals(pipeHardlinkBaseDirName);
+  }
+
+  /////////////////////////// Utils ///////////////////////////
+
+  private String findLongestDataDir(final @Nonnull String path) {
+    String matched = null;
+    for (final String dataDirPath : dataDirPathToPipeBaseDirPath.keySet()) {
+      if (path.startsWith(dataDirPath + File.separator)
+          && (Objects.isNull(matched) || dataDirPath.length() > 
matched.length())) {
+        matched = dataDirPath;
+      }
+    }
+    return matched;
+  }
+
+  private static String canonicalPath(final @Nonnull File file) throws 
IOException {
+    // getCanonicalPath is called on the File itself, so that an overridden 
implementation (and its
+    // failure) is neither lost nor turned into a non-canonical path.
+    try {
+      return file.getCanonicalPath();
+    } catch (final IOException e) {
+      throw failedToGetPipeDir(file, e);
+    }
+  }
+
+  /**
+   * Returns the canonical path of a configured data directory, falling back 
to its absolute path if
+   * it can not be resolved. The configuration of a directory must not fail 
the pin of a file.
+   */
+  private static String canonicalDataDirPath(final @Nonnull String dataDir) {
+    final File dataDirFile = new File(dataDir);
+    try {
+      return dataDirFile.getCanonicalPath();
+    } catch (final IOException e) {
+      return dataDirFile.getAbsolutePath();
+    }
+  }
+
+  private static IOException failedToGetPipeDir(
+      final @Nonnull File file, final @Nullable IOException cause) {
+    return new IOException(
+        String.format(
+            DataNodePipeMessages
+                
.PIPE_EXCEPTION_FAILED_TO_GET_HARDLINK_OR_COPIED_FILE_IN_PIPE_DIR_FOR_FILE_F009D86E,
+            file.getPath()),
+        cause);
+  }
+}
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileDirPathResolver.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileDirPathResolver.java
new file mode 100644
index 00000000000..ec3057e774a
--- /dev/null
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileDirPathResolver.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.resource.tsfile;
+
+import org.apache.iotdb.commons.pipe.config.PipeConfig;
+import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+/**
+ * Resolves the pipe directory that holds the hardlinked or copied files of a 
TsFile. The directory
+ * is always on the disk of the TsFile, see {@link PipeTsFileDirPathMapping}.
+ *
+ * <p>{@code dn_data_dirs} is hot reloadable, so the mapping follows the 
current local data
+ * directories instead of the ones seen at startup.
+ */
+public class PipeTsFileDirPathResolver {
+
+  private static volatile PipeTsFileDirPathResolver instance;
+
+  private final Supplier<String[]> localDataDirsSupplier;
+  private final String pipeHardlinkBaseDirName;
+  private final String pipeHardlinkTsFileDirName;
+
+  private volatile String[] resolvedLocalDataDirs;
+  private volatile PipeTsFileDirPathMapping mapping;
+
+  private PipeTsFileDirPathResolver(final @Nonnull Supplier<String[]> 
localDataDirsSupplier) {
+    this.localDataDirsSupplier = localDataDirsSupplier;
+    this.pipeHardlinkBaseDirName = 
PipeConfig.getInstance().getPipeHardlinkBaseDirName();
+    this.pipeHardlinkTsFileDirName = 
PipeConfig.getInstance().getPipeHardlinkTsFileDirName();
+
+    this.resolvedLocalDataDirs = localDataDirsSupplier.get();
+    this.mapping =
+        new PipeTsFileDirPathMapping(
+            resolvedLocalDataDirs, pipeHardlinkBaseDirName, 
pipeHardlinkTsFileDirName);
+  }
+
+  @TestOnly
+  PipeTsFileDirPathResolver() {
+    this(() -> new String[0]);
+  }
+
+  public static PipeTsFileDirPathResolver getInstance() {
+    if (Objects.isNull(instance)) {
+      synchronized (PipeTsFileDirPathResolver.class) {
+        if (Objects.isNull(instance)) {
+          instance =
+              new PipeTsFileDirPathResolver(
+                  () -> 
IoTDBDescriptor.getInstance().getConfig().getLocalDataDirs());
+        }
+      }
+    }
+    return instance;
+  }
+
+  @TestOnly
+  public static void setInstanceForTest(final @Nonnull Supplier<String[]> 
localDataDirsSupplier) {
+    instance = new PipeTsFileDirPathResolver(localDataDirsSupplier);
+  }
+
+  @TestOnly
+  public static void reset() {
+    instance = null;
+  }
+
+  /**
+   * Returns the hardlink or copied file of {@code file} in the pipe directory 
of {@code pipeName}.
+   */
+  static File getHardlinkOrCopiedFileInPipeDir(final File file, final 
@Nullable String pipeName)
+      throws IOException {
+    return getInstance().getHardlinkOrCopiedFile(file, pipeName);
+  }
+
+  File getHardlinkOrCopiedFile(final File file, final @Nullable String 
pipeName)
+      throws IOException {
+    return refresh(localDataDirsSupplier.get()).getHardlinkOrCopiedFile(file, 
pipeName);
+  }
+
+  /** Rebuilds the mapping if the local data directories have been changed by 
a hot reload. */
+  @Nonnull
+  PipeTsFileDirPathMapping refresh(final @Nonnull String[] localDataDirs) {
+    if (Arrays.equals(localDataDirs, resolvedLocalDataDirs)) {
+      return mapping;
+    }
+
+    synchronized (this) {
+      if (!Arrays.equals(localDataDirs, resolvedLocalDataDirs)) {
+        mapping =
+            new PipeTsFileDirPathMapping(
+                localDataDirs, pipeHardlinkBaseDirName, 
pipeHardlinkTsFileDirName);
+        resolvedLocalDataDirs = localDataDirs;
+      }
+      return mapping;
+    }
+  }
+}
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
index 02898d17b4b..6fd7bb8f305 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
@@ -19,8 +19,6 @@
 
 package org.apache.iotdb.db.pipe.resource.tsfile;
 
-import org.apache.iotdb.commons.conf.IoTDBConstant;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
 import org.apache.iotdb.commons.utils.FileUtils;
 import org.apache.iotdb.commons.utils.TestOnly;
 import org.apache.iotdb.db.i18n.DataNodePipeMessages;
@@ -56,7 +54,8 @@ public class PipeTsFileResourceManager {
   // PipeName -> TsFilePath -> PipeTsFileResource
   private final Map<String, Map<String, PipeTsFileResource>>
       hardlinkOrCopiedFileToPipeTsFileResourceMap = new ConcurrentHashMap<>();
-  private final Map<String, String> pipeNameToPipeTsFileDirPathMap = new 
ConcurrentHashMap<>();
+  // PipeName -> the pipe directories that hold the pipe's files, a pipe may 
span multiple disks
+  private final Map<String, Set<String>> pipeNameToPipeTsFileDirPathMap = new 
ConcurrentHashMap<>();
   private final Set<String> pipeTsFileResourcePipeNameSetUnderDeletion =
       ConcurrentHashMap.newKeySet();
   private final PipeTsFileResourceSegmentLock segmentLock = new 
PipeTsFileResourceSegmentLock();
@@ -68,46 +67,34 @@ public class PipeTsFileResourceManager {
 
   public File increaseFileReference(
       final File file, final boolean isTsFile, final @Nullable String 
pipeName) throws IOException {
-    return increaseFileReference(file, isTsFile, pipeName, null);
+    return increaseFileReference(file, isTsFile, pipeName, null, null);
   }
 
   /**
-   * Given a file, create a hardlink or copy it to pipe dir, maintain a 
reference count for the
-   * hardlink or copied file, and return the hardlink or copied file.
+   * Returns the hardlink or copied file of {@code file} in the pipe directory 
of {@code pipeName},
+   * creating it and increasing its reference count if it does not exist yet. 
A {@code null} pipe
+   * name means the public pipe file shared by the dataRegion -> assigner 
pipeline.
    *
-   * <p>If the given file is already a hardlink or copied file, increase its 
reference count and
-   * return it.
+   * <p>TsFiles are hardlinked, or copied if the hardlink can not be created, 
while .resource and
+   * .mod files are always copied.
    *
-   * <p>If the given file is a tsfile, create a hardlink in pipe dir, increase 
the reference count
-   * of the hardlink and return it.
-   *
-   * <p>Otherwise, copy the file (.mod or .resource) to pipe dir, increase the 
reference count of
-   * the copied file and return it.
-   *
-   * @param file tsfile, resource file or mod file. can be original file or 
hardlink/copy of
-   *     original file
-   * @param isTsFile {@code true} to create hardlink, {@code false} to copy 
file
-   * @param pipeName Nonnull if the pipe is from historical or assigner -> 
extractors, null if is
-   *     dataRegion -> assigner
-   * @param sourceFile for inner use, historical extractor will use this to 
create hardlink from
-   *     pipe tsFile -> common tsFile
-   * @return the hardlink or copied file
-   * @throws IOException when create hardlink or copy file failed
+   * @param sourceFile the file to hardlink or copy, or {@code null} to use 
{@code file}
+   * @param resource the source resource of {@code file}, or {@code null} if 
it is already final
+   * @throws IOException if the hardlink or the copy can not be created
    */
   private File increaseFileReference(
       final File file,
       final boolean isTsFile,
       final @Nullable String pipeName,
-      final @Nullable File sourceFile)
+      final @Nullable File sourceFile,
+      final @Nullable TsFileResource resource)
       throws IOException {
-    // If the file is already a hardlink or copied file,
-    // just increase reference count and return it
+    // The file is already a pipe file, so only its reference count has to be 
increased.
     if (increaseReferenceIfExists(file, pipeName, isTsFile)) {
       return file;
     }
 
-    // If the file is not a hardlink or copied file, check if there is a 
related hardlink or
-    // copied file in pipe dir. if so, increase reference count and return it
+    // The pipe file may exist without being tracked, e.g. left by a previous 
process.
     final File hardlinkOrCopiedFile =
         Objects.isNull(sourceFile) ? getHardlinkOrCopiedFileInPipeDir(file, 
pipeName) : file;
 
@@ -115,8 +102,7 @@ public class PipeTsFileResourceManager {
       return 
getResourceMap(pipeName).get(hardlinkOrCopiedFile.getPath()).getFile();
     }
 
-    // If the file is a tsfile, create a hardlink in pipe dir and will return 
it.
-    // otherwise, copy the file (.mod or .resource) to pipe dir and will 
return it.
+    // Create the pipe file and the resource that tracks its reference count.
     final File source = Objects.isNull(sourceFile) ? file : sourceFile;
     final File resultFile;
 
@@ -130,13 +116,14 @@ public class PipeTsFileResourceManager {
       } else {
         resultFile =
             isTsFile
-                ? FileUtils.createHardLink(source, hardlinkOrCopiedFile)
-                : FileUtils.copyFile(source, hardlinkOrCopiedFile);
+                ? createHardLinkOrCopy(source, hardlinkOrCopiedFile, resource)
+                : FileUtils.copyFileAtomically(source, hardlinkOrCopiedFile);
 
-        // Create the hardlink or copy and its reference-counted resource only 
when none exists.
+        // Track the new pipe file, and record its directory so that dropping 
the pipe cleans up.
         if (Objects.nonNull(pipeName)) {
-          pipeNameToPipeTsFileDirPathMap.putIfAbsent(
-              pipeName, hardlinkOrCopiedFile.getParentFile().getPath());
+          pipeNameToPipeTsFileDirPathMap
+              .computeIfAbsent(pipeName, k -> ConcurrentHashMap.newKeySet())
+              .add(hardlinkOrCopiedFile.getParentFile().getPath());
           hardlinkOrCopiedFileToPipeTsFileResourceMap
               .computeIfAbsent(pipeName, k -> new ConcurrentHashMap<>())
               .put(resultFile.getPath(), new PipeTsFileResource(resultFile));
@@ -175,9 +162,8 @@ public class PipeTsFileResourceManager {
     try {
       increasePublicReference(file, pipeName, isTsFile);
     } catch (final IOException | RuntimeException e) {
-      // The private reference is acquired before the public (assigner) 
reference.  If the latter
-      // fails, roll back the reference acquired above; otherwise every failed 
retry permanently
-      // pins the pipe file in the logical memory/file pool.
+      // A private reference must not outlive a failed public reference, 
otherwise every failed
+      // retry pins the pipe file for good.
       rollbackFileReference(file, pipeName, e);
       throw e;
     }
@@ -200,52 +186,12 @@ public class PipeTsFileResourceManager {
     }
     // Increase the assigner's file to avoid hard-link or memory cache cleaning
     // Note that it does not exist for historical files
-    increaseFileReference(new File(getCommonFilePath(file)), isTsFile, null, 
file);
+    increaseFileReference(new File(getCommonFilePath(file)), isTsFile, null, 
file, null);
   }
 
   public static File getHardlinkOrCopiedFileInPipeDir(
       final File file, final @Nullable String pipeName) throws IOException {
-    try {
-      return new File(getPipeTsFileDirPath(file, pipeName), 
getRelativeFilePath(file));
-    } catch (final Exception e) {
-      throw new IOException(
-          String.format(
-              DataNodePipeMessages
-                  
.PIPE_EXCEPTION_FAILED_TO_GET_HARDLINK_OR_COPIED_FILE_IN_PIPE_DIR_FOR_FILE_F009D86E,
-              file.getPath()),
-          e);
-    }
-  }
-
-  private static String getPipeTsFileDirPath(File file, final @Nullable String 
pipeName)
-      throws IOException {
-    while (!file.getName().equals(IoTDBConstant.SEQUENCE_FOLDER_NAME)
-        && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)
-        && 
!file.getName().equals(PipeConfig.getInstance().getPipeHardlinkBaseDirName())) {
-      file = file.getParentFile();
-    }
-    return file.getParentFile().getCanonicalPath()
-        + File.separator
-        + PipeConfig.getInstance().getPipeHardlinkBaseDirName()
-        + File.separator
-        + PipeConfig.getInstance().getPipeHardlinkTsFileDirName()
-        + (Objects.nonNull(pipeName) ? File.separator + pipeName : "");
-  }
-
-  private static String getRelativeFilePath(File file) {
-    StringBuilder builder = new StringBuilder(file.getName());
-    while (!file.getName().equals(IoTDBConstant.SEQUENCE_FOLDER_NAME)
-        && !file.getName().equals(IoTDBConstant.UNSEQUENCE_FOLDER_NAME)
-        && !file.getParentFile()
-            .getName()
-            .equals(PipeConfig.getInstance().getPipeHardlinkTsFileDirName())) {
-      file = file.getParentFile();
-      builder =
-          new StringBuilder(file.getName())
-              .append(IoTDBConstant.FILE_NAME_SEPARATOR)
-              .append(builder);
-    }
-    return builder.toString();
+    return PipeTsFileDirPathResolver.getHardlinkOrCopiedFileInPipeDir(file, 
pipeName);
   }
 
   /**
@@ -306,17 +252,20 @@ public class PipeTsFileResourceManager {
   }
 
   public void cleanPipeTsFileDir(final @Nonnull String 
pipeTsFileResourcePipeName) {
-    final String pipeTsFileDirPath =
+    final Set<String> pipeTsFileDirPaths =
         pipeNameToPipeTsFileDirPathMap.remove(pipeTsFileResourcePipeName);
     
hardlinkOrCopiedFileToPipeTsFileResourceMap.remove(pipeTsFileResourcePipeName);
 
-    if (Objects.isNull(pipeTsFileDirPath)) {
+    if (Objects.isNull(pipeTsFileDirPaths)) {
       
pipeTsFileResourcePipeNameSetUnderDeletion.remove(pipeTsFileResourcePipeName);
       return;
     }
 
-    
PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.submitStalePipeDirForPeriodicalCleanup(
-        new File(pipeTsFileDirPath));
+    // The files of one pipe may be scattered on multiple disks, clean every 
directory of the pipe.
+    for (final String pipeTsFileDirPath : pipeTsFileDirPaths) {
+      
PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.submitStalePipeDirForPeriodicalCleanup(
+          new File(pipeTsFileDirPath));
+    }
     
pipeTsFileResourcePipeNameSetUnderDeletion.remove(pipeTsFileResourcePipeName);
   }
 
@@ -431,7 +380,8 @@ public class PipeTsFileResourceManager {
   public boolean pinTsFileResource(
       final TsFileResource resource, final boolean withMods, final @Nullable 
String pipeName)
       throws IOException {
-    final File pinnedTsFile = increaseFileReference(resource.getTsFile(), 
true, pipeName);
+    final File pinnedTsFile =
+        increaseFileReference(resource.getTsFile(), true, pipeName, null, 
resource);
     try {
       if (withMods && resource.getExclusiveModFile().exists()) {
         increaseFileReference(resource.getExclusiveModFile().getFile(), false, 
pipeName);
@@ -461,10 +411,8 @@ public class PipeTsFileResourceManager {
       firstException = e;
     }
 
-    // Always attempt the mod-file cleanup even when resolving/decreasing the 
TsFile fails.  A
-    // failed first cleanup must not strand the second reference.
-    // Use the state captured when pinning. Compaction can remove the original 
mod file, and
-    // an event can acquire mod-file references even when no mod file was 
originally pinned.
+    // Clean up the mod file even if the TsFile failed, and only if it was 
pinned: compaction can
+    // remove the original mod file.
     if (pinnedModFile) {
       try {
         decreaseFileReference(
@@ -487,6 +435,50 @@ public class PipeTsFileResourceManager {
     }
   }
 
+  /**
+   * Hardlinks the source file into the pipe directory, or copies it if the 
hardlink can not be
+   * created, e.g. across disks. A TsFile which is not closed yet is never 
copied, because the copy
+   * would be truncated: the pin fails instead, and the TsFile is extracted 
after it is closed.
+   *
+   * @param resource the source resource of the file, or {@code null} if the 
file is already final
+   */
+  private static File createHardLinkOrCopy(
+      final File sourceFile,
+      final File hardlinkOrCopiedFile,
+      final @Nullable TsFileResource resource)
+      throws IOException {
+    if 
(sourceFile.getAbsoluteFile().equals(hardlinkOrCopiedFile.getAbsoluteFile())
+        && !hardlinkOrCopiedFile.isFile()) {
+      // The pipe file of this scope is gone, e.g. after drop pipe moved its 
directory aside.
+      throw new IOException(
+          String.format(
+              DataNodePipeMessages
+                  
.EXCEPTION_FAILED_TO_PIN_THE_PIPE_FILE_ARG_BECAUSE_IT_DOES_NOT_EXIST_1741B8E4,
+              hardlinkOrCopiedFile.getPath()));
+    }
+
+    // An unclosed TsFile may still be flushing. Copying it would pin a 
truncated file, so only a
+    // hardlink is acceptable. The same path is an existing pipe file and is 
returned as is.
+    if (Objects.nonNull(resource)
+        && !resource.isClosed()
+        && 
!sourceFile.getAbsoluteFile().equals(hardlinkOrCopiedFile.getAbsoluteFile())) {
+      try {
+        return FileUtils.createHardLink(sourceFile, hardlinkOrCopiedFile);
+      } catch (final IOException | UnsupportedOperationException linkFailure) {
+        final IOException unclosedException =
+            new IOException(
+                String.format(
+                    DataNodePipeMessages
+                        
.EXCEPTION_FAILED_TO_PIN_THE_TSFILE_ARG_BY_CREATING_A_HARDLINK_AND_THE_TSFILE_CAN_NOT_BE_COPIED_BECAUSE_IT_IS_NOT_CLOSED_YET_4E346D68,
+                    sourceFile.getPath()));
+        unclosedException.addSuppressed(linkFailure);
+        throw unclosedException;
+      }
+    }
+
+    return FileUtils.createHardLinkOrCopy(sourceFile, hardlinkOrCopiedFile);
+  }
+
   public int getLinkedTsFileCount(final @Nonnull String pipeName) {
     return hardlinkOrCopiedFileToPipeTsFileResourceMap
         .computeIfAbsent(pipeName, pipe -> new ConcurrentHashMap<>())
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
index b823f0fe2bf..f5d5f70b087 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
@@ -23,7 +23,9 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
 import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.path.MeasurementPath;
 import org.apache.iotdb.commons.utils.FileUtils;
+import org.apache.iotdb.db.i18n.DataNodePipeMessages;
 import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileDirPathResolver;
 import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
 import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
 import 
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
@@ -44,6 +46,7 @@ import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 import java.io.File;
 import java.io.IOException;
@@ -159,6 +162,8 @@ public class PipeTsFileResourceManagerTest {
 
   @After
   public void tearDown() throws Exception {
+    PipeTsFileDirPathResolver.reset();
+
     File pipeFolder = new File(ROOT_DIR);
     if (pipeFolder.exists()) {
       FileUtils.deleteFileOrDirectory(pipeFolder);
@@ -377,4 +382,235 @@ public class PipeTsFileResourceManagerTest {
       Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
     }
   }
+
+  @Test
+  public void testUnclosedTsFileIsNotCopiedWhenHardlinkFails() throws 
Exception {
+    final File originTsFile = new File(TS_FILE_NAME);
+    final File pipeTsFileDir =
+        
PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(originTsFile, 
PIPE_NAME)
+            .getParentFile();
+    // Block the pipe dir with a regular file, so that the hardlink fails and 
the copy is taken.
+    final File blockingFile = pipeTsFileDir.getParentFile();
+    Assert.assertTrue(blockingFile.getParentFile().mkdirs());
+    Assert.assertTrue(blockingFile.createNewFile());
+
+    final TsFileResource unclosedResource = Mockito.mock(TsFileResource.class);
+    Mockito.when(unclosedResource.getTsFile()).thenReturn(originTsFile);
+    Mockito.when(unclosedResource.isClosed()).thenReturn(false);
+
+    // A copy of an open TsFile would be truncated, so the pin must fail and 
be retried later.
+    final String unclosedMessage =
+        String.format(
+            DataNodePipeMessages
+                
.EXCEPTION_FAILED_TO_PIN_THE_TSFILE_ARG_BY_CREATING_A_HARDLINK_AND_THE_TSFILE_CAN_NOT_BE_COPIED_BECAUSE_IT_IS_NOT_CLOSED_YET_4E346D68,
+            originTsFile.getPath());
+    try {
+      pipeTsFileResourceManager.pinTsFileResource(unclosedResource, false, 
PIPE_NAME);
+      Assert.fail("The TsFile which is not closed yet should not be copied");
+    } catch (final IOException e) {
+      Assert.assertEquals(unclosedMessage, e.getMessage());
+      // The reason why the hardlink failed is kept as a suppressed exception.
+      Assert.assertEquals(1, e.getSuppressed().length);
+      Assert.assertTrue(e.getSuppressed()[0] instanceof IOException);
+    }
+
+    // A closed TsFile does fall back to copying, which fails only because the 
dir is blocked.
+    final TsFileResource closedResource = Mockito.mock(TsFileResource.class);
+    Mockito.when(closedResource.getTsFile()).thenReturn(originTsFile);
+    Mockito.when(closedResource.isClosed()).thenReturn(true);
+    try {
+      pipeTsFileResourceManager.pinTsFileResource(closedResource, false, 
PIPE_NAME);
+      Assert.fail("The pipe directory is blocked by a file, so the copy should 
fail");
+    } catch (final IOException e) {
+      Assert.assertNotEquals(unclosedMessage, e.getMessage());
+    }
+    // Nothing must have been pinned or copied.
+    Assert.assertFalse(pipeTsFileDir.toPath().toFile().exists());
+  }
+
+  @Test
+  public void testExistingPipeFileIsReusedInsteadOfCopiedOntoItself() throws 
Exception {
+    final File originTsFile = new File(TS_FILE_NAME);
+    final File pipeTsFile =
+        
PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(originTsFile, 
PIPE_NAME);
+    Assert.assertTrue(pipeTsFile.getParentFile().mkdirs());
+    // Simulate a pipe file which has been left by a previous process and is 
not tracked yet.
+    FileUtils.copyFile(originTsFile, pipeTsFile);
+
+    // The target is the source here, so a failed pin must not delete it.
+    final File referencedFile =
+        pipeTsFileResourceManager.increaseFileReference(pipeTsFile, true, 
PIPE_NAME);
+    Assert.assertEquals(pipeTsFile, referencedFile);
+    Assert.assertTrue(pipeTsFile.exists());
+    Assert.assertEquals(
+        1, pipeTsFileResourceManager.getFileReferenceCount(referencedFile, 
PIPE_NAME));
+
+    pipeTsFileResourceManager.decreaseFileReference(referencedFile, PIPE_NAME);
+    Assert.assertEquals(0, 
pipeTsFileResourceManager.getFileReferenceCount(pipeTsFile, PIPE_NAME));
+    Assert.assertFalse(pipeTsFile.exists());
+  }
+
+  @Test
+  public void testPinTsFileUnderConfiguredDataDirKeepsItsOwnDisk() throws 
Exception {
+    final File dataDir = new File(ROOT_DIR, "data").getCanonicalFile();
+    final File tsFile =
+        new File(dataDir, IoTDBConstant.SEQUENCE_FOLDER_NAME + File.separator 
+ "test.tsfile");
+    createTsfile(tsFile.getPath());
+    PipeTsFileDirPathResolver.setInstanceForTest(() -> new String[] 
{dataDir.getPath()});
+
+    final File pipeTsFile =
+        PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(tsFile, 
PIPE_NAME);
+    Assert.assertEquals(
+        new File(
+            new File(new File(dataDir, "pipe"), "tsfile"),
+            PIPE_NAME + File.separator + "sequence-test.tsfile"),
+        pipeTsFile);
+
+    pipeTsFileResourceManager.pinTsFileResource(closedResourceOf(tsFile), 
false, PIPE_NAME);
+    Assert.assertTrue(pipeTsFile.isFile());
+    Assert.assertEquals(tsFile.length(), pipeTsFile.length());
+
+    pipeTsFileResourceManager.unpinTsFileResource(closedResourceOf(tsFile), 
PIPE_NAME, false);
+    Assert.assertFalse(pipeTsFile.exists());
+  }
+
+  @Test
+  public void testCleanPipeTsFileDirCleansEveryDataDir() throws Exception {
+    final File dataDir1 = new File(ROOT_DIR, "data1").getCanonicalFile();
+    final File dataDir2 = new File(ROOT_DIR, "data2").getCanonicalFile();
+    final File tsFile1 =
+        new File(dataDir1, IoTDBConstant.SEQUENCE_FOLDER_NAME + File.separator 
+ "f1.tsfile");
+    final File tsFile2 =
+        new File(dataDir2, IoTDBConstant.SEQUENCE_FOLDER_NAME + File.separator 
+ "f2.tsfile");
+    createTsfile(tsFile1.getPath());
+    createTsfile(tsFile2.getPath());
+    PipeTsFileDirPathResolver.setInstanceForTest(
+        () -> new String[] {dataDir1.getPath(), dataDir2.getPath()});
+
+    pipeTsFileResourceManager.pinTsFileResource(closedResourceOf(tsFile1), 
false, PIPE_NAME);
+    pipeTsFileResourceManager.pinTsFileResource(closedResourceOf(tsFile2), 
false, PIPE_NAME);
+    final File pipeTsFileDir1 =
+        PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(tsFile1, 
PIPE_NAME)
+            .getParentFile();
+    final File pipeTsFileDir2 =
+        PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(tsFile2, 
PIPE_NAME)
+            .getParentFile();
+    Assert.assertNotEquals(pipeTsFileDir1, pipeTsFileDir2);
+    Assert.assertTrue(pipeTsFileDir1.isDirectory());
+    Assert.assertTrue(pipeTsFileDir2.isDirectory());
+
+    pipeTsFileResourceManager.cleanPipeTsFileDir(PIPE_NAME);
+
+    Assert.assertFalse(pipeTsFileDir1.exists());
+    Assert.assertFalse(pipeTsFileDir2.exists());
+    Assert.assertTrue(pipeTsFileDir1.getParentFile().isDirectory());
+    Assert.assertTrue(pipeTsFileDir2.getParentFile().isDirectory());
+  }
+
+  @Test
+  public void testPinSucceedsOnRetryAfterTheHardlinkFailed() throws Exception {
+    final File originTsFile = new File(TS_FILE_NAME);
+    final File pipeTsFile =
+        
PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(originTsFile, 
PIPE_NAME);
+    final File blockingFile = pipeTsFile.getParentFile().getParentFile();
+    Assert.assertTrue(blockingFile.getParentFile().mkdirs());
+    Assert.assertTrue(blockingFile.createNewFile());
+
+    try {
+      
pipeTsFileResourceManager.pinTsFileResource(closedResourceOf(originTsFile), 
false, PIPE_NAME);
+      Assert.fail("The pipe directory is blocked by a file, so the pin should 
fail");
+    } catch (final IOException e) {
+      // Expected: the failed pin must not leave anything behind.
+      Assert.assertFalse(pipeTsFile.exists());
+      Assert.assertFalse(new File(pipeTsFile.getPath() + 
".unfinished").exists());
+    }
+
+    Assert.assertTrue(blockingFile.delete());
+    
pipeTsFileResourceManager.pinTsFileResource(closedResourceOf(originTsFile), 
false, PIPE_NAME);
+    Assert.assertTrue(pipeTsFile.isFile());
+    Assert.assertEquals(originTsFile.length(), pipeTsFile.length());
+    Assert.assertFalse(new File(pipeTsFile.getPath() + 
".unfinished").exists());
+
+    
pipeTsFileResourceManager.unpinTsFileResource(closedResourceOf(originTsFile), 
PIPE_NAME, false);
+    Assert.assertFalse(pipeTsFile.exists());
+  }
+
+  @Test
+  public void testStalePipeFileWithDifferentContentIsReplaced() throws 
Exception {
+    final File originTsFile = new File(TS_FILE_NAME);
+    final File pipeTsFile =
+        
PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(originTsFile, 
PIPE_NAME);
+    Assert.assertTrue(pipeTsFile.getParentFile().mkdirs());
+    // A stale pipe file, e.g. left by a killed process: it is replaced, not 
reused.
+    Files.write(pipeTsFile.toPath(), new byte[] {1, 2, 3});
+
+    
pipeTsFileResourceManager.pinTsFileResource(closedResourceOf(originTsFile), 
false, PIPE_NAME);
+
+    Assert.assertEquals(originTsFile.length(), pipeTsFile.length());
+  }
+
+  @Test
+  public void testStaleCopiedFileIsReplacedInsteadOfFailingThePin() throws 
Exception {
+    final File originModFile = new File(MODS_FILE_NAME);
+    final File pipeModFile =
+        
PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(originModFile, 
PIPE_NAME);
+    Assert.assertTrue(pipeModFile.getParentFile().mkdirs());
+    // A partial copy and the temporary file of a failure that has been killed 
before.
+    Files.write(pipeModFile.toPath(), new byte[] {1, 2, 3});
+    Files.write(new File(pipeModFile.getPath() + ".unfinished").toPath(), new 
byte[] {4});
+
+    final TsFileResource resource = closedResourceOf(new File(TS_FILE_NAME));
+    final ModificationFile modFile = Mockito.mock(ModificationFile.class);
+    Mockito.when(modFile.exists()).thenReturn(true);
+    Mockito.when(modFile.getFile()).thenReturn(originModFile);
+    Mockito.when(resource.getExclusiveModFile()).thenReturn(modFile);
+
+    Assert.assertTrue(pipeTsFileResourceManager.pinTsFileResource(resource, 
true, PIPE_NAME));
+
+    Assert.assertEquals(originModFile.length(), pipeModFile.length());
+    Assert.assertFalse(new File(pipeModFile.getPath() + 
".unfinished").exists());
+  }
+
+  @Test
+  public void testAtomicCopyLeavesNothingBehindWhenItFails() throws Exception {
+    final File targetFile =
+        PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+            new File(MODS_FILE_NAME), PIPE_NAME);
+    try {
+      FileUtils.copyFileAtomically(new File(ROOT_DIR, "missing.mod"), 
targetFile);
+      Assert.fail("Copying a missing file should fail");
+    } catch (final IOException e) {
+      Assert.assertFalse(targetFile.exists());
+      Assert.assertFalse(new File(targetFile.getPath() + 
".unfinished").exists());
+    }
+  }
+
+  @Test
+  public void testMissingPipeFileFailsThePin() throws Exception {
+    final File pipeTsFile =
+        PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+            new File(TS_FILE_NAME), PIPE_NAME);
+    // An event pinned after the pipe directory has been moved aside must not 
be reported as pinned.
+    final File missingPipeFile = new File(pipeTsFile.getParentFile(), 
"missing.tsfile");
+    try {
+      pipeTsFileResourceManager.increaseFileReference(missingPipeFile, true, 
PIPE_NAME);
+      Assert.fail("The pipe file does not exist, so the pin should fail");
+    } catch (final IOException e) {
+      Assert.assertEquals(
+          String.format(
+              DataNodePipeMessages
+                  
.EXCEPTION_FAILED_TO_PIN_THE_PIPE_FILE_ARG_BECAUSE_IT_DOES_NOT_EXIST_1741B8E4,
+              missingPipeFile.getPath()),
+          e.getMessage());
+    }
+    Assert.assertEquals(
+        0, pipeTsFileResourceManager.getFileReferenceCount(missingPipeFile, 
PIPE_NAME));
+  }
+
+  private static TsFileResource closedResourceOf(final File tsFile) {
+    final TsFileResource resource = Mockito.mock(TsFileResource.class);
+    Mockito.when(resource.getTsFile()).thenReturn(tsFile);
+    Mockito.when(resource.isClosed()).thenReturn(true);
+    return resource;
+  }
 }
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileDirPathMappingTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileDirPathMappingTest.java
new file mode 100644
index 00000000000..14e0797b0c2
--- /dev/null
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileDirPathMappingTest.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.resource.tsfile;
+
+import org.apache.iotdb.commons.utils.FileUtils;
+import org.apache.iotdb.db.i18n.DataNodePipeMessages;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public class PipeTsFileDirPathMappingTest {
+
+  private Path rootDir;
+
+  private String dataDir1;
+  private String dataDir2;
+
+  @Before
+  public void setUp() throws Exception {
+    rootDir =
+        Files.createTempDirectory("pipe-hardlink-dir-mapping-test")
+            .toFile()
+            .getCanonicalFile()
+            .toPath();
+    dataDir1 = new File(rootDir.toFile(), "data1").getCanonicalPath();
+    dataDir2 = new File(rootDir.toFile(), "data2").getCanonicalPath();
+  }
+
+  @After
+  public void tearDown() {
+    FileUtils.deleteFileOrDirectory(rootDir.toFile(), true);
+  }
+
+  @Test
+  public void testEveryDataDirUsesItsOwnPipeDir() throws Exception {
+    // Every data dir keeps its pipe dir on its own disk.
+    final PipeTsFileDirPathMapping mapping =
+        new PipeTsFileDirPathMapping(new String[] {dataDir1, dataDir2}, 
"pipe", "tsfile");
+
+    Assert.assertEquals(
+        new File(
+            new File(new File(new File(dataDir1, "pipe"), "tsfile"), "pipe1"),
+            "sequence-root.db-0-0-1-1-0-0.tsfile"),
+        mapping.getHardlinkOrCopiedFile(
+            new File(dataDir1, "sequence/root.db/0/0/1-1-0-0.tsfile"), 
"pipe1"));
+
+    Assert.assertEquals(
+        new File(
+            new File(new File(new File(dataDir2, "pipe"), "tsfile"), "pipe1"),
+            "unsequence-root.db-0-0-f2.tsfile"),
+        mapping.getHardlinkOrCopiedFile(
+            new File(dataDir2, "unsequence/root.db/0/0/f2.tsfile"), "pipe1"));
+  }
+
+  @Test
+  public void testPipeDirOutsideDataDirsIsCanonical() throws Exception {
+    // A symlinked ancestor must not yield two paths for one physical pipe 
file.
+    final File nonCanonicalRoot = 
Files.createTempDirectory("pipe-hardlink-canonical").toFile();
+    try {
+      final PipeTsFileDirPathMapping mapping =
+          new PipeTsFileDirPathMapping(new String[0], "pipe", "tsfile");
+
+      final File resolved =
+          mapping.getHardlinkOrCopiedFile(
+              new File(nonCanonicalRoot, "sequence" + File.separator + 
"f.tsfile"), null);
+
+      Assert.assertEquals(resolved.getCanonicalPath(), resolved.getPath());
+      
Assert.assertTrue(resolved.getPath().startsWith(nonCanonicalRoot.getCanonicalPath()));
+    } finally {
+      FileUtils.deleteFileOrDirectory(nonCanonicalRoot, true);
+    }
+  }
+
+  @Test
+  public void testFileOutsideDataDirsFallsBackToLegacyLayout() throws 
Exception {
+    final PipeTsFileDirPathMapping mapping =
+        new PipeTsFileDirPathMapping(new String[] {dataDir1}, "pipe", 
"tsfile");
+
+    final String otherDataDir = new File(rootDir.toFile(), 
"data3").getCanonicalPath();
+    Assert.assertEquals(
+        new File(
+            new File(new File(new File(otherDataDir, "pipe"), "tsfile"), 
"pipe1"),
+            "sequence-root.db-0-0-f3.tsfile"),
+        mapping.getHardlinkOrCopiedFile(
+            new File(otherDataDir, "sequence/root.db/0/0/f3.tsfile"), 
"pipe1"));
+  }
+
+  @Test
+  public void testPublicPipeFileIsLinkedIntoTheRequestingPipe() throws 
Exception {
+    final PipeTsFileDirPathMapping mapping =
+        new PipeTsFileDirPathMapping(new String[] {dataDir1}, "pipe", 
"tsfile");
+
+    final File tsFile = new File(dataDir1, "sequence/root.db/0/0/f7.tsfile");
+
+    // The assigner pins the TsFile into the public pipe directory.
+    final File publicPipeFile = mapping.getHardlinkOrCopiedFile(tsFile, null);
+    Assert.assertEquals(
+        new File(new File(dataDir1, "pipe/tsfile"), 
"sequence-root.db-0-0-f7.tsfile"),
+        publicPipeFile);
+
+    // Every pipe gets a file of its own, so that it can not delete the 
assigner's file.
+    final File pipeFile = mapping.getHardlinkOrCopiedFile(publicPipeFile, 
"pipe1");
+    Assert.assertEquals(
+        new File(new File(dataDir1, "pipe/tsfile/pipe1"), 
"sequence-root.db-0-0-f7.tsfile"),
+        pipeFile);
+    Assert.assertNotEquals(
+        publicPipeFile, mapping.getHardlinkOrCopiedFile(publicPipeFile, 
"pipe2"));
+
+    // A file of the asking scope is reused, never nested.
+    Assert.assertEquals(publicPipeFile, 
mapping.getHardlinkOrCopiedFile(publicPipeFile, null));
+    Assert.assertEquals(pipeFile, mapping.getHardlinkOrCopiedFile(pipeFile, 
"pipe1"));
+  }
+
+  @Test
+  public void testFileAlreadyInPipeDirIsReturnedAsIs() throws Exception {
+    final PipeTsFileDirPathMapping mapping =
+        new PipeTsFileDirPathMapping(new String[] {dataDir1}, "pipe", 
"tsfile");
+
+    final File pipeTsFile =
+        new File(new File(new File(dataDir1, "pipe"), "tsfile/pipe1"), 
"f5.tsfile");
+    Assert.assertEquals(pipeTsFile, 
mapping.getHardlinkOrCopiedFile(pipeTsFile, "pipe1"));
+  }
+
+  @Test
+  public void testMappingIsRefreshedWhenDataDirsAreHotReloaded() throws 
Exception {
+    final PipeTsFileDirPathResolver resolver = new PipeTsFileDirPathResolver();
+
+    final PipeTsFileDirPathMapping mappingBeforeReload = resolver.refresh(new 
String[] {dataDir1});
+    // Unchanged data dirs must reuse the cached mapping.
+    Assert.assertSame(mappingBeforeReload, resolver.refresh(new String[] 
{dataDir1}));
+
+    // A disk appended by a hot reload of dn_data_dirs must be taken into 
account.
+    final PipeTsFileDirPathMapping mappingAfterReload =
+        resolver.refresh(new String[] {dataDir1, dataDir2});
+    Assert.assertNotSame(mappingBeforeReload, mappingAfterReload);
+    Assert.assertEquals(
+        new File(
+            new File(new File(new File(dataDir2, "pipe"), "tsfile"), "pipe1"),
+            "sequence-root.db-0-0-f6.tsfile"),
+        mappingAfterReload.getHardlinkOrCopiedFile(
+            new File(dataDir2, "sequence/root.db/0/0/f6.tsfile"), "pipe1"));
+  }
+
+  @Test
+  public void testFileInAnExistingPipeDirIsResolvedFromTheLayout() throws 
Exception {
+    final PipeTsFileDirPathMapping mapping =
+        new PipeTsFileDirPathMapping(new String[] {dataDir1}, "pipe", 
"tsfile");
+
+    // A leftover pipe file outside the data dirs, in a pipe dir named like 
the pipe base dir.
+    final String otherDataDir = new File(rootDir.toFile(), 
"data3").getCanonicalPath();
+    final File pipeFile =
+        new File(new File(new File(new File(otherDataDir, "pipe"), "tsfile"), 
"pipe"), "f8.tsfile");
+    Assert.assertEquals(pipeFile, mapping.getHardlinkOrCopiedFile(pipeFile, 
"pipe"));
+
+    // A file of another pipe is linked into the asking pipe, not adopted by 
it.
+    Assert.assertEquals(
+        new File(new File(new File(otherDataDir, "pipe"), "tsfile/pipe2"), 
"f8.tsfile"),
+        mapping.getHardlinkOrCopiedFile(pipeFile, "pipe2"));
+  }
+
+  @Test
+  public void testUnresolvableFileFailsInsteadOfUsingItsAbsolutePath() throws 
Exception {
+    final PipeTsFileDirPathMapping mapping =
+        new PipeTsFileDirPathMapping(new String[] {dataDir1}, "pipe", 
"tsfile");
+    final File unresolvableFile =
+        new UnresolvableFile(new File(dataDir1, 
"sequence/root.db/0/0/f9.tsfile").getPath());
+
+    // The failure must not be swallowed: a pin on a non-canonical path would 
make one physical file
+    // look like two, so the resolution fails instead.
+    try {
+      mapping.getHardlinkOrCopiedFile(unresolvableFile, "pipe1");
+      Assert.fail("The path which can not be resolved should fail the 
resolution");
+    } catch (final IOException e) {
+      Assert.assertEquals(
+          String.format(
+              DataNodePipeMessages
+                  
.PIPE_EXCEPTION_FAILED_TO_GET_HARDLINK_OR_COPIED_FILE_IN_PIPE_DIR_FOR_FILE_F009D86E,
+              unresolvableFile.getPath()),
+          e.getMessage());
+      Assert.assertTrue(e.getCause() instanceof IOException);
+    }
+  }
+
+  private static class UnresolvableFile extends File {
+
+    private UnresolvableFile(final String path) {
+      super(path);
+    }
+
+    @Override
+    public String getCanonicalPath() throws IOException {
+      throw new IOException("mock path resolution failure");
+    }
+  }
+}
diff --git 
a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java
 
b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java
index 0bbb7b10f13..401950ce9a3 100644
--- 
a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java
+++ 
b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/UtilMessages.java
@@ -79,6 +79,9 @@ public final class UtilMessages {
       "failed to create hardlink %s for file %s: failed to create parent dir 
%s";
   public static final String FAILED_TO_COPY_FILE_PARENT_DIR =
       "failed to copy file %s to %s: failed to create parent dir %s";
+  public static final String
+      
LOG_FAILED_TO_CREATE_THE_HARDLINK_OF_THE_FILE_ARG_TO_ARG_FALLING_BACK_TO_COPYING_THE_FILE_89FED96B
 =
+          "Failed to create the hardlink of the file {} to {}, falling back to 
copying the file";
   public static final String DELETED_DUPLICATE_FILE =
       "Deleted the file {} because it already exists in the target directory: 
{}";
   public static final String FAILED_TO_CREATE_TARGET_DIRECTORY =
diff --git 
a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java
 
b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java
index f08332078ef..b52260322b0 100644
--- 
a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java
+++ 
b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/UtilMessages.java
@@ -77,6 +77,9 @@ public final class UtilMessages {
       "创建硬链接 %s 失败(源文件 %s):无法创建父目录 %s";
   public static final String FAILED_TO_COPY_FILE_PARENT_DIR =
       "复制文件 %s 到 %s 失败:无法创建父目录 %s";
+  public static final String
+      
LOG_FAILED_TO_CREATE_THE_HARDLINK_OF_THE_FILE_ARG_TO_ARG_FALLING_BACK_TO_COPYING_THE_FILE_89FED96B
 =
+          "为文件 {} 创建到 {} 的硬链接失败,改为拷贝该文件";
   public static final String DELETED_DUPLICATE_FILE =
       "已删除文件 {},因为目标目录 {} 中已存在同名文件";
   public static final String FAILED_TO_CREATE_TARGET_DIRECTORY =
diff --git 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
index 78b3a4fbda5..b3aead57c68 100644
--- 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
+++ 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java
@@ -36,6 +36,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.channels.FileChannel;
+import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.DirectoryNotEmptyException;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.FileSystems;
@@ -59,6 +60,7 @@ public class FileUtils {
   private static final int BUFFER_SIZE = 1024;
   private static final long FILE_REMOVE_COST_IN_BYTES = 64 * 1024L;
   private static final long DIRECTORY_REMOVE_COST_IN_BYTES = 4 * 1024L;
+  private static final String UNFINISHED_FILE_SUFFIX = ".unfinished";
 
   private FileUtils() {}
 
@@ -479,6 +481,39 @@ public class FileUtils {
     return hardlink;
   }
 
+  /**
+   * Hardlinks {@code sourceFile} to {@code hardlinkOrCopiedFile}, or copies 
it atomically when the
+   * hardlink can not be created, e.g. across disks. The same file is returned 
as is and is never
+   * linked onto itself.
+   */
+  public static File createHardLinkOrCopy(final File sourceFile, final File 
hardlinkOrCopiedFile)
+      throws IOException {
+    if 
(sourceFile.getAbsoluteFile().equals(hardlinkOrCopiedFile.getAbsoluteFile())) {
+      // Never link a file onto itself: a failed link would delete the target, 
which is the source.
+      if (!hardlinkOrCopiedFile.isFile()) {
+        throw new NoSuchFileException(hardlinkOrCopiedFile.getPath());
+      }
+      return hardlinkOrCopiedFile;
+    }
+
+    try {
+      return createHardLink(sourceFile, hardlinkOrCopiedFile);
+    } catch (final IOException | UnsupportedOperationException linkFailure) {
+      LOGGER.warn(
+          UtilMessages
+              
.LOG_FAILED_TO_CREATE_THE_HARDLINK_OF_THE_FILE_ARG_TO_ARG_FALLING_BACK_TO_COPYING_THE_FILE_89FED96B,
+          sourceFile.getAbsolutePath(),
+          hardlinkOrCopiedFile.getAbsolutePath(),
+          linkFailure);
+      try {
+        return copyFileAtomically(sourceFile, hardlinkOrCopiedFile);
+      } catch (final IOException | UnsupportedOperationException copyFailure) {
+        copyFailure.addSuppressed(linkFailure);
+        throw copyFailure;
+      }
+    }
+  }
+
   public static File copyFile(File sourceFile, File targetFile) throws 
IOException {
     if (!targetFile.getParentFile().exists() && 
!targetFile.getParentFile().mkdirs()) {
       synchronized (FileUtils.class) {
@@ -497,6 +532,35 @@ public class FileUtils {
     return targetFile;
   }
 
+  /**
+   * Copies {@code sourceFile} to {@code targetFile} by writing a temporary 
file first and then
+   * renaming it, so that a crash or a full disk can never leave a truncated 
file under the final
+   * name. An existing target, including a partial copy left by a previous 
failure, is replaced.
+   */
+  public static File copyFileAtomically(final File sourceFile, final File 
targetFile)
+      throws IOException {
+    final File unfinishedFile = new File(targetFile.getPath() + 
UNFINISHED_FILE_SUFFIX);
+    try {
+      // A temporary file left by a killed process would fail the copy the 
same way.
+      deleteFileIfExist(unfinishedFile);
+      copyFile(sourceFile, unfinishedFile);
+      try {
+        Files.move(
+            unfinishedFile.toPath(),
+            targetFile.toPath(),
+            StandardCopyOption.ATOMIC_MOVE,
+            StandardCopyOption.REPLACE_EXISTING);
+      } catch (final AtomicMoveNotSupportedException e) {
+        Files.move(
+            unfinishedFile.toPath(), targetFile.toPath(), 
StandardCopyOption.REPLACE_EXISTING);
+      }
+      return targetFile;
+    } catch (final IOException | RuntimeException e) {
+      deleteFileIfExist(unfinishedFile);
+      throw e;
+    }
+  }
+
   /**
    * Transfer bytes to human-readable string. Copy from <a
    * href="https://stackoverflow.com/a/3758880";>stackoverflow</a>.

Reply via email to