This is an automated email from the ASF dual-hosted git repository.

Caideyipi pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/dev/1.3 by this push:
     new 3dce926e6df Prevent deleted values from reappearing after a local LOAD 
crash (#19136) (#19188)
3dce926e6df is described below

commit 3dce926e6df265f858ba2ada04e46b1a6be4766a
Author: Caideyipi <[email protected]>
AuthorDate: Mon Nov 9 14:12:13 2026 +0800

    Prevent deleted values from reappearing after a local LOAD crash (#19136) 
(#19188)
    
    * Prevent partial local LOAD installation after process crashes
    
    * Keep committed LOAD successful when source cleanup fails
    
    * Avoid counting converted LOAD mods twice
    
    * Record LOAD mods metrics only after publication
    
    * Align LOAD cleanup tests with source group retention
    
    (cherry picked from commit c6e229aef50608cb67bfb4fd94717d05f4ef3d0f)
---
 .../iotdb/it/env/cluster/config/MppJVMConfig.java  |  11 +
 .../it/env/cluster/node/AbstractNodeWrapper.java   |   5 +
 .../iotdb/db/it/IoTDBLoadTsFileInstallIT.java      | 422 +++++++++++++++++++++
 .../org/apache/iotdb/db/it/utils/NodeDebugger.java | 232 +++++++++++
 .../iotdb/db/i18n/DataNodeQueryMessages.java       |   2 +
 .../iotdb/db/i18n/StorageEngineMessages.java       |  48 +++
 .../iotdb/db/i18n/DataNodeQueryMessages.java       |   1 +
 .../iotdb/db/i18n/StorageEngineMessages.java       |  46 +++
 .../plan/node/load/LoadSingleTsFileNode.java       |  12 +-
 .../db/storageengine/dataregion/DataRegion.java    | 248 +++++++-----
 .../plan/planner/node/load/LoadTsFileNodeTest.java |  34 +-
 .../storageengine/dataregion/DataRegionTest.java   | 144 +++++++
 12 files changed, 1107 insertions(+), 98 deletions(-)

diff --git 
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
 
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
index d9c51717cf8..7e46af79736 100644
--- 
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
+++ 
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
@@ -29,6 +29,16 @@ public class MppJVMConfig implements JVMConfig {
   private int maxHeapSize;
   private int maxDirectMemorySize;
   private String timezone;
+  private int debugPort;
+
+  /** Enables loopback-only JDWP for tests that need to observe an exact node 
execution stage. */
+  public void setDebugPort(int debugPort) {
+    this.debugPort = debugPort;
+  }
+
+  public int getDebugPort() {
+    return debugPort;
+  }
 
   @Override
   public JVMConfig setInitHeapSize(int initSize) {
@@ -94,6 +104,7 @@ public class MppJVMConfig implements JVMConfig {
     this.setMaxHeapSize(config.getMaxHeapSize());
     this.setMaxDirectMemorySize(config.getMaxDirectMemorySize());
     this.setTimezone(config.getTimezone());
+    this.setDebugPort(config.getDebugPort());
   }
 
   public static Builder builder() {
diff --git 
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
 
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
index afad120e94c..701c19de817 100644
--- 
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
+++ 
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
@@ -416,6 +416,11 @@ public abstract class AbstractNodeWrapper implements 
BaseNodeWrapper {
       File stdoutFile = new File(getLogPath());
       List<String> startCmd = new ArrayList<>();
       startCmd.add(JAVA_CMD);
+      if (jvmConfig.getDebugPort() > 0) {
+        startCmd.add(
+            
"-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=127.0.0.1:"
+                + jvmConfig.getDebugPort());
+      }
       if (!SystemUtils.IS_JAVA_1_8) {
         startCmd.add("--add-opens=java.base/java.util.concurrent=ALL-UNNAMED");
         startCmd.add("--add-opens=java.base/java.lang=ALL-UNNAMED");
diff --git 
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileInstallIT.java
 
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileInstallIT.java
new file mode 100644
index 00000000000..46e1fd5bc98
--- /dev/null
+++ 
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileInstallIT.java
@@ -0,0 +1,422 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.it.utils.NodeDebugger;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.config.MppJVMConfig;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+
+import com.sun.jdi.ObjectReference;
+import com.sun.jdi.StackFrame;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.PlainDeviceID;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.awaitility.Awaitility;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.net.ServerSocket;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/** A single-replica process fixture for the local whole-file installation 
commit boundary. */
+@RunWith(IoTDBTestRunner.class)
+@Category(ClusterIT.class)
+public class IoTDBLoadTsFileInstallIT {
+  private static final String DATABASE = "root.load.install";
+  private static final String DATA_REGION =
+      "org.apache.iotdb.db.storageengine.dataregion.DataRegion";
+  private static final long[] TIMES = {-3, -2, -1};
+  private static final List<MeasurementSchema> SCHEMAS =
+      Arrays.asList(
+          new MeasurementSchema("s", TSDataType.INT64),
+          new MeasurementSchema("txt", TSDataType.TEXT));
+  private static int debugPort;
+  private File directory;
+
+  @BeforeClass
+  public static void setUpClass() throws Exception {
+    try (ServerSocket socket = new ServerSocket(0)) {
+      debugPort = socket.getLocalPort();
+    }
+    ((MppJVMConfig) 
EnvFactory.getEnv().getConfig().getDataNodeJVMConfig()).setDebugPort(debugPort);
+    EnvFactory.getEnv()
+        .getConfig()
+        .getCommonConfig()
+        .setDataReplicationFactor(1)
+        .setSchemaReplicationFactor(1)
+        .setDataRegionConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS)
+        .setDefaultDataRegionGroupNumPerDatabase(1)
+        .setDataRegionGroupExtensionPolicy("CUSTOM");
+    EnvFactory.getEnv().initClusterEnvironment(1, 1);
+  }
+
+  @AfterClass
+  public static void tearDownClass() {
+    try {
+      EnvFactory.getEnv().cleanClusterEnvironment();
+    } finally {
+      ((MppJVMConfig) 
EnvFactory.getEnv().getConfig().getDataNodeJVMConfig()).setDebugPort(0);
+    }
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    directory = Files.createTempDirectory("load-install-").toFile();
+    execute("create database " + DATABASE);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    try {
+      if (!node().isAlive()) {
+        startNode();
+      }
+      execute("delete database " + DATABASE);
+    } finally {
+      try (Stream<Path> paths = Files.walk(directory.toPath())) {
+        for (Path path : 
paths.sorted(Comparator.reverseOrder()).collect(Collectors.toList())) {
+          Files.delete(path);
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testCrashBeforeDeletionInstallationDoesNotPublishPartialFile() 
throws Exception {
+    assertCrashBoundary(false, false);
+  }
+
+  @Test
+  public void testCrashAfterDeletionInstallationDoesNotPublishPartialFile() 
throws Exception {
+    assertCrashBoundary(false, true);
+  }
+
+  @Test
+  public void testPublishedFileSurvivesCrashBeforeClientAcknowledgement() 
throws Exception {
+    assertCrashBoundary(true, true);
+  }
+
+  @Test
+  public void testSourceCleanupFailureDoesNotFailCommittedLoad() throws 
Exception {
+    for (boolean failDataDeletion : new boolean[] {false, true}) {
+      final String prefix = DATABASE + ".cleanup_data" + failDataDeletion;
+      final File source = new File(directory, (200 + (failDataDeletion ? 2 : 
0)) + "-0-0-0.tsfile");
+      writeSource(source, prefix);
+      final File sourceResource = new File(source + 
TsFileResource.RESOURCE_SUFFIX);
+      final File sourceMods = new File(source + ModificationFile.FILE_SUFFIX);
+      final byte[] sourceData = Files.readAllBytes(source.toPath());
+      final ExecutorService executor = Executors.newSingleThreadExecutor();
+      try {
+        try (NodeDebugger debugger =
+            new NodeDebugger(debugPort, TsFileManager.class.getName(), "add")) 
{
+          final Future<?> request =
+              executor.submit(
+                  () -> {
+                    execute(load(source, true));
+                    return null;
+                  });
+          final StackFrame frame =
+              debugger.await(
+                  candidate -> {
+                    final ObjectReference resource =
+                        (ObjectReference) candidate.getArgumentValues().get(0);
+                    return NodeDebugger.filePath(
+                            
resource.getValue(resource.referenceType().fieldByName("file")))
+                        .contains(File.separator + DATABASE + File.separator);
+                  });
+          final ObjectReference resource = (ObjectReference) 
frame.getArgumentValues().get(0);
+          final File target =
+              new File(
+                  NodeDebugger.filePath(
+                      
resource.getValue(resource.referenceType().fieldByName("file"))));
+          debugger.pauseOnExit();
+          debugger.resume();
+          debugger.await(candidate -> true);
+          Assert.assertFalse(
+              "The original SQL must still be awaiting its response", 
request.isDone());
+          // Registration is complete, and DataRegion has not started source 
cleanup yet.
+          assertSource(source, true);
+          Assert.assertTrue(target.isFile());
+          Assert.assertTrue(new File(target + 
TsFileResource.RESOURCE_SUFFIX).isFile());
+          Assert.assertTrue(new File(target + 
ModificationFile.FILE_SUFFIX).isFile());
+          Assert.assertFalse(new File(target + ".load-pending").exists());
+
+          // A nonempty directory makes deleteIfExists fail on every supported 
platform.
+          final File blocked = failDataDeletion ? source : sourceResource;
+          Files.delete(blocked.toPath());
+          Files.createDirectory(blocked.toPath());
+          Files.write(
+              blocked.toPath().resolve("retained"), failDataDeletion ? 
sourceData : new byte[] {1});
+          debugger.resume();
+          request.get(30, TimeUnit.SECONDS);
+        }
+      } finally {
+        executor.shutdownNow();
+        Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
+      }
+
+      if (failDataDeletion) {
+        Assert.assertTrue(source.isDirectory());
+        Assert.assertArrayEquals(
+            sourceData, 
Files.readAllBytes(source.toPath().resolve("retained")));
+        Assert.assertTrue(sourceResource.isFile());
+        Assert.assertTrue(sourceMods.isFile());
+      } else {
+        Assert.assertFalse(source.exists());
+        Assert.assertTrue(sourceResource.isDirectory());
+        Assert.assertTrue(new File(sourceResource, "retained").isFile());
+        // SQL batch cleanup can remove the remaining mods once the source 
data is gone.
+        Assert.assertFalse(sourceMods.exists());
+      }
+      assertRows(prefix);
+    }
+  }
+
+  private void assertCrashBoundary(boolean published, boolean 
deletionsInstalled) throws Exception {
+    for (boolean delete : new boolean[] {false, true}) {
+      final String prefix = DATABASE + ".d" + delete;
+      final File source = new File(directory, (100 + (delete ? 2 : 0)) + 
"-0-0-0.tsfile");
+      writeSource(source, prefix);
+      final ExecutorService executor = Executors.newSingleThreadExecutor();
+      final File target;
+      final boolean visibleBeforeCrash;
+      final Future<?> request;
+      try {
+        try (NodeDebugger debugger =
+            new NodeDebugger(
+                debugPort, DATA_REGION, published ? "loadTsFileToUnSequence" : 
"loadModFile")) {
+          request =
+              executor.submit(
+                  () -> {
+                    execute(load(source, delete));
+                    return null;
+                  });
+          StackFrame frame =
+              debugger.await(
+                  candidate ->
+                      
NodeDebugger.filePath(candidate.getArgumentValues().get(0))
+                          .equals(source.getAbsolutePath()));
+          if (published) {
+            debugger.pauseOnExit();
+            debugger.resume();
+            frame = debugger.await(candidate -> true);
+            final ObjectReference resource = (ObjectReference) 
frame.getArgumentValues().get(1);
+            target =
+                new File(
+                    NodeDebugger.filePath(
+                        
resource.getValue(resource.referenceType().fieldByName("file"))));
+            Assert.assertTrue(
+                "Published file must already include all deletions",
+                new File(target + ModificationFile.FILE_SUFFIX).isFile());
+          } else {
+            target = new 
File(NodeDebugger.filePath(frame.getArgumentValues().get(1)));
+            if (deletionsInstalled) {
+              debugger.pauseOnExit();
+              debugger.resume();
+              debugger.await(candidate -> true);
+            }
+            Assert.assertEquals(
+                deletionsInstalled, new File(target + 
ModificationFile.FILE_SUFFIX).isFile());
+          }
+          Assert.assertTrue(new File(target + 
TsFileResource.RESOURCE_SUFFIX).isFile());
+          visibleBeforeCrash = target.isFile();
+          Assert.assertFalse(
+              "The original SQL must still be awaiting its response", 
request.isDone());
+          node().stopForcibly();
+          final ExecutionException failure =
+              Assert.assertThrows(
+                  ExecutionException.class, () -> request.get(30, 
TimeUnit.SECONDS));
+          Assert.assertTrue(failure.toString(), failure.getCause() instanceof 
SQLException);
+        }
+        // The commit can complete its source cleanup before the client 
receives a response.
+        assertSource(source, !published || !delete);
+        startNode();
+        if (published) {
+          Assert.assertTrue(visibleBeforeCrash);
+          assertRows(prefix);
+        } else {
+          // Check recovery before any new LOAD; retrying first could conceal 
resurrection.
+          try (Connection connection = EnvFactory.getEnv().getConnection();
+              Statement statement = connection.createStatement();
+              ResultSet rows =
+                  statement.executeQuery("select s from " + prefix + ".d0 
where time=-2")) {
+            Assert.assertFalse("Recovery must not resurrect a deleted value", 
rows.next());
+          }
+          Assert.assertFalse(
+              "A file without its deletions must never be published", 
visibleBeforeCrash);
+          try (Connection connection = EnvFactory.getEnv().getConnection();
+              Statement statement = connection.createStatement();
+              ResultSet rows = statement.executeQuery("select ** from " + 
prefix + ".**")) {
+            Assert.assertFalse("Unpublished data must stay invisible after 
recovery", rows.next());
+          }
+          Assert.assertFalse(target.exists());
+          Assert.assertFalse(new File(target + ".load-pending").exists());
+          Assert.assertFalse(new File(target + 
TsFileResource.RESOURCE_SUFFIX).exists());
+          Assert.assertFalse(new File(target + 
ModificationFile.FILE_SUFFIX).exists());
+          // This is an explicit new submission, not recovery of the 
interrupted request.
+          execute(load(source, delete));
+          assertRows(prefix);
+          assertSource(source, !delete);
+        }
+      } finally {
+        executor.shutdownNow();
+        Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
+      }
+    }
+  }
+
+  private static void writeSource(File source, String prefix) throws Exception 
{
+    try (TsFileWriter writer = new TsFileWriter(source)) {
+      for (int device = 0; device < 2; device++) {
+        final String path = prefix + ".d" + device;
+        writer.registerAlignedTimeseries(new 
org.apache.tsfile.read.common.Path(path), SCHEMAS);
+        final Tablet tablet = new Tablet(path, SCHEMAS, TIMES.length);
+        for (int row = 0; row < TIMES.length; row++) {
+          tablet.addTimestamp(row, TIMES[row]);
+          tablet.addValue("s", row, device * 100L + row);
+          tablet.addValue("txt", row, row == 0 ? null : "d" + device + "_r" + 
row);
+        }
+        tablet.rowSize = TIMES.length;
+        writer.writeAligned(tablet);
+      }
+    }
+    final TsFileResource resource = new TsFileResource(source);
+    for (int device = 0; device < 2; device++) {
+      final IDeviceID id = new PlainDeviceID(prefix + ".d" + device);
+      resource.updateStartTime(id, TIMES[0]);
+      resource.updateEndTime(id, TIMES[TIMES.length - 1]);
+    }
+    resource.serialize();
+    try (ModificationFile mods = resource.getModFile()) {
+      for (int device = 0; device < 2; device++) {
+        mods.write(
+            new Deletion(
+                new MeasurementPath(prefix + ".d" + device + ".s"),
+                Long.MAX_VALUE,
+                TIMES[1],
+                TIMES[1]));
+      }
+    }
+  }
+
+  private static void assertRows(String prefix) throws Exception {
+    for (int device = 0; device < 2; device++) {
+      try (Connection connection = EnvFactory.getEnv().getConnection();
+          Statement statement = connection.createStatement();
+          ResultSet rows = statement.executeQuery("select s,txt from " + 
prefix + ".d" + device)) {
+        for (int row = 0; row < TIMES.length; row++) {
+          Assert.assertTrue(rows.next());
+          Assert.assertEquals(TIMES[row], rows.getLong(1));
+          Assert.assertEquals(row == 1 ? null : device * 100L + row, 
rows.getObject(2));
+          Assert.assertEquals(row == 0 ? null : "d" + device + "_r" + row, 
rows.getString(3));
+        }
+        Assert.assertFalse(rows.next());
+      }
+    }
+  }
+
+  private static void assertSource(File source, boolean exists) {
+    Assert.assertEquals(exists, source.isFile());
+    Assert.assertEquals(exists, new File(source + 
TsFileResource.RESOURCE_SUFFIX).isFile());
+    Assert.assertEquals(exists, new File(source + 
ModificationFile.FILE_SUFFIX).isFile());
+  }
+
+  private static String load(File source, boolean delete) {
+    return "load '"
+        + source.getAbsolutePath()
+        + "' with 
('database-level'='2','tablet-conversion-threshold'='-1','on-success'='"
+        + (delete ? "delete" : "none")
+        + "')";
+  }
+
+  private static void execute(String sql) throws Exception {
+    try (Connection connection =
+            
EnvFactory.getEnv().getWriteOnlyConnectionWithSpecifiedDataNode(node());
+        Statement statement = connection.createStatement()) {
+      statement.execute(sql);
+    }
+  }
+
+  private static DataNodeWrapper node() {
+    return EnvFactory.getEnv().getDataNodeWrapperList().get(0);
+  }
+
+  private static void startNode() throws Exception {
+    node().start();
+    EnvFactory.getEnv()
+        .ensureNodeStatus(
+            Collections.singletonList(node()), 
Collections.singletonList(NodeStatus.Running));
+    Awaitility.await()
+        .atMost(60, TimeUnit.SECONDS)
+        .untilAsserted(
+            () -> {
+              try (Connection connection = EnvFactory.getEnv().getConnection();
+                  Statement statement = connection.createStatement();
+                  ResultSet regions = statement.executeQuery("show regions")) {
+                int count = 0;
+                while (regions.next()) {
+                  if (DATABASE.equals(regions.getString("Database"))
+                      && "DataRegion".equals(regions.getString("Type"))) {
+                    count++;
+                    Assert.assertEquals(
+                        NodeStatus.Running.getStatus(), 
regions.getString("Status"));
+                  }
+                }
+                Assert.assertTrue("The actual DataRegion must be ready after 
restart", count > 0);
+              }
+            });
+  }
+}
diff --git 
a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/NodeDebugger.java 
b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/NodeDebugger.java
new file mode 100644
index 00000000000..c37bd06f1a1
--- /dev/null
+++ 
b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/NodeDebugger.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it.utils;
+
+import com.sun.jdi.Bootstrap;
+import com.sun.jdi.ObjectReference;
+import com.sun.jdi.ReferenceType;
+import com.sun.jdi.StackFrame;
+import com.sun.jdi.StringReference;
+import com.sun.jdi.ThreadReference;
+import com.sun.jdi.VMDisconnectedException;
+import com.sun.jdi.Value;
+import com.sun.jdi.VirtualMachine;
+import com.sun.jdi.connect.AttachingConnector;
+import com.sun.jdi.connect.Connector;
+import com.sun.jdi.event.BreakpointEvent;
+import com.sun.jdi.event.ClassPrepareEvent;
+import com.sun.jdi.event.Event;
+import com.sun.jdi.event.EventSet;
+import com.sun.jdi.event.MethodEntryEvent;
+import com.sun.jdi.event.MethodExitEvent;
+import com.sun.jdi.event.VMStartEvent;
+import com.sun.jdi.request.BreakpointRequest;
+import com.sun.jdi.request.ClassPrepareRequest;
+import com.sun.jdi.request.EventRequest;
+import com.sun.jdi.request.MethodEntryRequest;
+import com.sun.jdi.request.MethodExitRequest;
+import org.junit.Assert;
+
+import java.net.ConnectException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+/**
+ * Pauses only the node thread executing a selected method. Other threads 
(including the active
+ * scanner) keep running. All control stays in the test JVM; no product hooks 
or agent bytecode are
+ * installed. Closing the debugger releases a paused thread even when an 
assertion fails.
+ */
+public final class NodeDebugger implements AutoCloseable {
+  private final VirtualMachine vm;
+  private final String className;
+  private final String methodName;
+  private final boolean observeMethodEntries;
+  private final Set<ReferenceType> breakpointTypes = new HashSet<>();
+  private EventSet suspended;
+  private ThreadReference pausedThread;
+
+  public NodeDebugger(int port, String className, String methodName) throws 
Exception {
+    this(port, className, methodName, false);
+  }
+
+  public NodeDebugger(int port, String className, String methodName, boolean 
observeMethodEntries)
+      throws Exception {
+    this.className = className;
+    this.methodName = methodName;
+    this.observeMethodEntries = observeMethodEntries;
+    final AttachingConnector connector =
+        Bootstrap.virtualMachineManager().attachingConnectors().stream()
+            .filter(candidate -> 
candidate.name().equals("com.sun.jdi.SocketAttach"))
+            .findFirst()
+            .orElseThrow(AssertionError::new);
+    final Map<String, Connector.Argument> arguments = 
connector.defaultArguments();
+    arguments.get("hostname").setValue("127.0.0.1");
+    arguments.get("port").setValue(Integer.toString(port));
+    arguments.get("timeout").setValue("10000");
+    final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+    VirtualMachine connected;
+    while (true) {
+      try {
+        connected = connector.attach(arguments);
+        break;
+      } catch (ConnectException e) {
+        if (System.nanoTime() >= deadline) {
+          throw e;
+        }
+        // ProcessBuilder.start returns before the suspended VM has opened its 
JDWP listener.
+        TimeUnit.MILLISECONDS.sleep(100);
+      }
+    }
+    vm = connected;
+    try {
+      if (observeMethodEntries) {
+        installMethodEntryRequest();
+      } else {
+        final ClassPrepareRequest prepare = 
vm.eventRequestManager().createClassPrepareRequest();
+        prepare.addClassFilter(className);
+        prepare.setSuspendPolicy(EventRequest.SUSPEND_EVENT_THREAD);
+        prepare.enable();
+        for (ReferenceType type : vm.classesByName(className)) {
+          if (type.isPrepared()) {
+            installBreakpoint(type);
+          }
+        }
+      }
+    } catch (Exception | AssertionError e) {
+      vm.dispose();
+      throw e;
+    }
+  }
+
+  private void installMethodEntryRequest() {
+    final MethodEntryRequest entry = 
vm.eventRequestManager().createMethodEntryRequest();
+    entry.addClassFilter(className);
+    entry.setSuspendPolicy(EventRequest.SUSPEND_EVENT_THREAD);
+    entry.enable();
+  }
+
+  /** Releases only the initial VMStart suspension after all requests have 
been installed. */
+  public void resumeStartup() throws Exception {
+    suspended = vm.eventQueue().remove(10000);
+    Assert.assertNotNull("Suspended startup must produce a VMStart event", 
suspended);
+    Assert.assertTrue(suspended.stream().anyMatch(event -> event instanceof 
VMStartEvent));
+    resume();
+  }
+
+  private void installBreakpoint(ReferenceType type) {
+    if (!breakpointTypes.add(type)) {
+      return;
+    }
+    Assert.assertEquals(className + "." + methodName, 1, 
type.methodsByName(methodName).size());
+    final BreakpointRequest breakpoint =
+        vm.eventRequestManager()
+            
.createBreakpointRequest(type.methodsByName(methodName).get(0).location());
+    breakpoint.setSuspendPolicy(EventRequest.SUSPEND_EVENT_THREAD);
+    breakpoint.enable();
+  }
+
+  public StackFrame await(Predicate<StackFrame> condition) throws Exception {
+    final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60);
+    while (System.nanoTime() < deadline) {
+      final EventSet events =
+          vm.eventQueue()
+              .remove(Math.max(1, TimeUnit.NANOSECONDS.toMillis(deadline - 
System.nanoTime())));
+      if (events == null) {
+        break;
+      }
+      // Remember ownership before inspecting a frame: failed predicates must 
also release it.
+      suspended = events;
+      for (Event event : events) {
+        if (event instanceof ClassPrepareEvent && !observeMethodEntries) {
+          installBreakpoint(((ClassPrepareEvent) event).referenceType());
+        } else if (event instanceof MethodEntryEvent
+            && ((MethodEntryEvent) event).method().name().equals(methodName)) {
+          final ThreadReference thread = ((MethodEntryEvent) event).thread();
+          final StackFrame frame = thread.frame(0);
+          if (condition.test(frame)) {
+            pausedThread = thread;
+            return frame;
+          }
+        } else if (event instanceof BreakpointEvent) {
+          final StackFrame frame = ((BreakpointEvent) event).thread().frame(0);
+          if (condition.test(frame)) {
+            pausedThread = ((BreakpointEvent) event).thread();
+            return frame;
+          }
+        } else if (event instanceof MethodExitEvent
+            && ((MethodExitEvent) event).method().name().equals(methodName)) {
+          final StackFrame frame = ((MethodExitEvent) event).thread().frame(0);
+          if (condition.test(frame)) {
+            // Resume must let cleanup and the SQL response run without 
another exit pause.
+            event.request().disable();
+            return frame;
+          }
+        }
+      }
+      resume();
+    }
+    throw new AssertionError("Did not reach " + className + "." + methodName);
+  }
+
+  public void resume() {
+    if (suspended != null) {
+      suspended.resume();
+      suspended = null;
+    }
+  }
+
+  /** Changes an entry pause into an exit pause on the same method and 
connection. */
+  public void pauseOnExit() {
+    vm.eventRequestManager()
+        .deleteEventRequests(
+            new 
java.util.ArrayList<>(vm.eventRequestManager().breakpointRequests()));
+    vm.eventRequestManager()
+        .deleteEventRequests(
+            new 
java.util.ArrayList<>(vm.eventRequestManager().classPrepareRequests()));
+    vm.eventRequestManager()
+        .deleteEventRequests(
+            new 
java.util.ArrayList<>(vm.eventRequestManager().methodEntryRequests()));
+    final MethodExitRequest exit = 
vm.eventRequestManager().createMethodExitRequest();
+    exit.addClassFilter(className);
+    Assert.assertNotNull("An entry pause is required before selecting its 
exit", pausedThread);
+    exit.addThreadFilter(pausedThread);
+    exit.setSuspendPolicy(EventRequest.SUSPEND_EVENT_THREAD);
+    exit.enable();
+  }
+
+  /** Reads java.io.File's path without invoking code in the paused node. */
+  public static String filePath(Value value) {
+    final ObjectReference file = (ObjectReference) value;
+    return ((StringReference) 
file.getValue(file.referenceType().fieldByName("path"))).value();
+  }
+
+  @Override
+  public void close() {
+    try {
+      resume();
+      vm.dispose();
+    } catch (VMDisconnectedException expectedAfterCrash) {
+      // A forced node stop is the operation under test.
+    }
+  }
+}
diff --git 
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index 0acc13f6cf5..e02a1aed7b0 100644
--- 
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++ 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -64,4 +64,6 @@ public final class DataNodeQueryMessages {
           "There is not enough memory for Query %s, the contextHolder is 
%s,current remaining free "
               + "memory is %dB, already reserved memory for this context in 
total is %dB, the memory "
               + "requested this time is %dB";
+  public static final String LOG_DELETE_AFTER_LOADING_ARG_ERROR_C606A060 =
+      "Delete After Loading {} error.";
 }
diff --git 
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java
 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java
new file mode 100644
index 00000000000..d799152d9b5
--- /dev/null
+++ 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/StorageEngineMessages.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.i18n;
+
+public final class StorageEngineMessages {
+
+  private StorageEngineMessages() {}
+
+  public static final String
+      
LOG_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_ARG_TARGET_ARG_13205CEC =
+          "File renaming failed when loading tsfile. Origin: {}, Target: {}";
+  public static final String
+      
EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_ARG_TARGET_ARG_BECAUSE_ARG_4A8CF51E
 =
+          "File renaming failed when loading tsfile. Origin: %s, Target: %s, 
because %s";
+  public static final String
+      
LOG_FILE_RENAMING_FAILED_WHEN_LOADING_RESOURCE_FILE_ORIGIN_ARG_TARGET_ARG_5589CD09
 =
+          "File renaming failed when loading .resource file. Origin: {}, 
Target: {}";
+  public static final String
+      
EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_RESOURCE_FILE_ORIGIN_ARG_TARGET_ARG_BECAUSE_ARG_77B1F6DE
 =
+          "File renaming failed when loading .resource file. Origin: %s, 
Target: %s, because %s";
+  public static final String
+      
LOG_FILE_RENAMING_FAILED_WHEN_LOADING_MOD_FILE_ORIGIN_ARG_TARGET_ARG_2493B8EE =
+          "File renaming failed when loading .mod file. Origin: {}, Target: 
{}";
+  public static final String
+      
EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_MOD_FILE_ORIGIN_ARG_TARGET_ARG_BECAUSE_ARG_93A853C4
 =
+          "File renaming failed when loading .mod file. Origin: %s, Target: 
%s, because %s";
+  public static final String LOG_CANNOT_DELETE_LOCALMODFILE_ARG_F9CD0EA6 =
+      "Cannot delete localModFile {}";
+  public static final String 
LOG_FAILED_TO_DELETE_FILE_OR_DIRECTORY_ARG_7EC21473 =
+      "Failed to delete file or directory: {}";
+}
diff --git 
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
index 72ea24db250..2aaee8bcfc7 100644
--- 
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
+++ 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
@@ -61,4 +61,5 @@ public final class DataNodeQueryMessages {
   public static final String
       
QUERY_EXCEPTION_THERE_IS_NOT_ENOUGH_MEMORY_FOR_QUERY_S_THE_CONTEXTHOLDER_546CDD02
 =
           "Query %s 内存不足,contextHolder 为 %s,当前剩余空闲内存为 %dB,该 context 已预留总内存为 
%dB,本次请求内存为 %dB。";
+  public static final String LOG_DELETE_AFTER_LOADING_ARG_ERROR_C606A060 = 
"加载后删除 {} 时出错。";
 }
diff --git 
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java
 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java
new file mode 100644
index 00000000000..6f43d0b34c8
--- /dev/null
+++ 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/StorageEngineMessages.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.i18n;
+
+public final class StorageEngineMessages {
+
+  private StorageEngineMessages() {}
+
+  public static final String
+      
LOG_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_ARG_TARGET_ARG_13205CEC =
+          "加载 TsFile 时重命名文件失败。源文件:{},目标文件:{}";
+  public static final String
+      
EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_ARG_TARGET_ARG_BECAUSE_ARG_4A8CF51E
 =
+          "加载 TsFile 时重命名文件失败。源文件:%s,目标文件:%s,原因:%s";
+  public static final String
+      
LOG_FILE_RENAMING_FAILED_WHEN_LOADING_RESOURCE_FILE_ORIGIN_ARG_TARGET_ARG_5589CD09
 =
+          "加载 .resource 文件时重命名失败。源文件:{},目标文件:{}";
+  public static final String
+      
EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_RESOURCE_FILE_ORIGIN_ARG_TARGET_ARG_BECAUSE_ARG_77B1F6DE
 =
+          "加载 .resource 文件时重命名失败。源文件:%s,目标文件:%s,原因:%s";
+  public static final String
+      
LOG_FILE_RENAMING_FAILED_WHEN_LOADING_MOD_FILE_ORIGIN_ARG_TARGET_ARG_2493B8EE =
+          "加载 .mod 文件时重命名失败。源文件:{},目标文件:{}";
+  public static final String
+      
EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_MOD_FILE_ORIGIN_ARG_TARGET_ARG_BECAUSE_ARG_93A853C4
 =
+          "加载 .mod 文件时重命名失败。源文件:%s,目标文件:%s,原因:%s";
+  public static final String LOG_CANNOT_DELETE_LOCALMODFILE_ARG_F9CD0EA6 = 
"无法删除本地 mods 文件 {}";
+  public static final String 
LOG_FAILED_TO_DELETE_FILE_OR_DIRECTORY_ARG_7EC21473 = "删除文件或目录失败:{}";
+}
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java
index b73068d1395..8784f197081 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
 import org.apache.iotdb.commons.utils.RetryUtils;
 import org.apache.iotdb.commons.utils.TimePartitionUtils;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
 import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
@@ -219,20 +220,25 @@ public class LoadSingleTsFileNode extends WritePlanNode {
     if (!deleteAfterLoad) {
       return;
     }
-    deleteFile(tsFile);
+    // A retained source TsFile still needs its sidecars for any later 
explicit LOAD.
+    if (!deleteFile(tsFile)) {
+      return;
+    }
     deleteFile(new File(tsFile.getAbsolutePath() + 
TsFileResource.RESOURCE_SUFFIX));
     deleteFile(new File(tsFile.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX));
   }
 
-  private void deleteFile(final File file) {
+  private boolean deleteFile(final File file) {
     try {
       RetryUtils.retryOnException(
           () -> {
             Files.deleteIfExists(file.toPath());
             return null;
           });
+      return true;
     } catch (final Exception e) {
-      LOGGER.warn("Delete After Loading {} error.", file, e);
+      
LOGGER.warn(DataNodeQueryMessages.LOG_DELETE_AFTER_LOADING_ARG_ERROR_C606A060, 
file, e);
+      return false;
     }
   }
 
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 986d06fa817..43043aa7668 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -51,6 +51,7 @@ import org.apache.iotdb.db.exception.load.LoadFileException;
 import org.apache.iotdb.db.exception.query.OutOfTTLException;
 import org.apache.iotdb.db.exception.query.QueryProcessException;
 import org.apache.iotdb.db.exception.quota.ExceedQuotaException;
+import org.apache.iotdb.db.i18n.StorageEngineMessages;
 import 
org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener;
 import org.apache.iotdb.db.queryengine.common.DeviceContext;
 import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
@@ -130,7 +131,6 @@ import org.apache.iotdb.db.utils.DateTimeUtils;
 import org.apache.iotdb.rpc.RpcUtils;
 import org.apache.iotdb.rpc.TSStatusCode;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.tsfile.file.metadata.ChunkMetadata;
 import org.apache.tsfile.file.metadata.IDeviceID;
 import org.apache.tsfile.file.metadata.PlainDeviceID;
@@ -150,6 +150,7 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -219,6 +220,7 @@ public class DataRegion implements IDataRegionForQuery {
   private static final int MERGE_MOD_START_VERSION_NUM = 1;
 
   private static final Logger logger = 
LoggerFactory.getLogger(DataRegion.class);
+  private static final String LOAD_PENDING_SUFFIX = ".load-pending";
 
   /**
    * A read write lock for guaranteeing concurrent safety when accessing all 
fields in this class
@@ -760,6 +762,14 @@ public class DataRegion implements IDataRegionForQuery {
           } else {
             // some TsFileResource may be being persisted when the system 
crashed, try recovering
             // such resources
+            final File[] pendingLoads =
+                fsFactory.listFilesBySuffix(
+                    partitionFolder.getAbsolutePath(), TSFILE_SUFFIX + 
LOAD_PENDING_SUFFIX);
+            if (pendingLoads != null) {
+              for (File pendingLoad : pendingLoads) {
+                deleteUnpublishedLoad(pendingLoad);
+              }
+            }
             continueFailedRenames(partitionFolder, TEMP_SUFFIX);
             String partitionName = partitionFolder.getName();
             File[] tsFilesInThisFolder =
@@ -801,6 +811,20 @@ public class DataRegion implements IDataRegionForQuery {
     }
   }
 
+  private void deleteUnpublishedLoad(File pendingFile) throws IOException {
+    final File targetFile =
+        new File(
+            pendingFile
+                .getPath()
+                .substring(0, pendingFile.getPath().length() - 
LOAD_PENDING_SUFFIX.length()));
+    if (!targetFile.exists()) {
+      Files.deleteIfExists(new File(targetFile + RESOURCE_SUFFIX).toPath());
+      Files.deleteIfExists(new File(targetFile + 
ModificationFile.FILE_SUFFIX).toPath());
+    }
+    // Keep the pending file until all sidecars are removed, so interrupted 
cleanup is retryable.
+    Files.deleteIfExists(pendingFile.toPath());
+  }
+
   /** check if the tsfile's time is smaller than system current time. */
   private void checkTsFileTime(File tsFile, long currentTime) throws 
DataRegionException {
     String[] items = tsFile.getName().replace(TSFILE_SUFFIX, 
"").split(FILE_NAME_SEPARATOR);
@@ -3535,116 +3559,93 @@ public class DataRegion implements IDataRegionForQuery 
{
     if (!targetFile.getParentFile().exists()) {
       targetFile.getParentFile().mkdirs();
     }
-    try {
-      if (deleteOriginFile) {
-        RetryUtils.retryOnException(
-            () -> {
-              FileUtils.moveFile(tsFileToLoad, targetFile);
-              return null;
-            });
-      } else {
-        RetryUtils.retryOnException(
-            () -> {
-              Files.copy(tsFileToLoad.toPath(), targetFile.toPath());
-              return null;
-            });
-      }
-    } catch (final IOException e) {
-      logger.warn(
-          "File renaming failed when loading tsfile. Origin: {}, Target: {}",
-          tsFileToLoad.getAbsolutePath(),
-          targetFile.getAbsolutePath(),
-          e);
-      throw new LoadFileException(
-          String.format(
-              "File renaming failed when loading tsfile. Origin: %s, Target: 
%s, because %s",
-              tsFileToLoad.getAbsolutePath(), targetFile.getAbsolutePath(), 
e.getMessage()));
-    }
-
+    // Recovery only discovers names ending in .tsfile. Keep the data 
invisible until its resource
+    // and every deletion are installed, including while the data is being 
copied.
+    final File pendingFile = new File(targetFile + LOAD_PENDING_SUFFIX);
     final File resourceFileToLoad =
         fsFactory.getFile(tsFileToLoad.getAbsolutePath() + RESOURCE_SUFFIX);
     final File targetResourceFile =
         fsFactory.getFile(targetFile.getAbsolutePath() + RESOURCE_SUFFIX);
+    boolean loadPublished = false;
     try {
-      if (deleteOriginFile) {
+      try {
+        if (pendingFile.exists()) {
+          deleteUnpublishedLoad(pendingFile);
+        }
         RetryUtils.retryOnException(
             () -> {
-              FileUtils.moveFile(resourceFileToLoad, targetResourceFile);
+              Files.copy(
+                  tsFileToLoad.toPath(), pendingFile.toPath(), 
StandardCopyOption.REPLACE_EXISTING);
               return null;
             });
-      } else {
+      } catch (final IOException e) {
+        logger.warn(
+            StorageEngineMessages
+                
.LOG_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_ARG_TARGET_ARG_13205CEC,
+            tsFileToLoad.getAbsolutePath(),
+            targetFile.getAbsolutePath(),
+            e);
+        throw new LoadFileException(
+            String.format(
+                StorageEngineMessages
+                    
.EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_ARG_TARGET_ARG_BECAUSE_ARG_4A8CF51E,
+                tsFileToLoad.getAbsolutePath(),
+                targetFile.getAbsolutePath(),
+                e.getMessage()));
+      }
+
+      try {
         RetryUtils.retryOnException(
             () -> {
-              Files.copy(resourceFileToLoad.toPath(), 
targetResourceFile.toPath());
+              Files.copy(
+                  resourceFileToLoad.toPath(),
+                  targetResourceFile.toPath(),
+                  StandardCopyOption.REPLACE_EXISTING);
               return null;
             });
+      } catch (final IOException e) {
+        logger.warn(
+            StorageEngineMessages
+                
.LOG_FILE_RENAMING_FAILED_WHEN_LOADING_RESOURCE_FILE_ORIGIN_ARG_TARGET_ARG_5589CD09,
+            resourceFileToLoad.getAbsolutePath(),
+            targetResourceFile.getAbsolutePath(),
+            e);
+        throw new LoadFileException(
+            String.format(
+                StorageEngineMessages
+                    
.EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_RESOURCE_FILE_ORIGIN_ARG_TARGET_ARG_BECAUSE_ARG_77B1F6DE,
+                resourceFileToLoad.getAbsolutePath(),
+                targetResourceFile.getAbsolutePath(),
+                e.getMessage()));
       }
-    } catch (final IOException e) {
-      logger.warn(
-          "File renaming failed when loading .resource file. Origin: {}, 
Target: {}",
-          resourceFileToLoad.getAbsolutePath(),
-          targetResourceFile.getAbsolutePath(),
-          e);
-      throw new LoadFileException(
-          String.format(
-              "File renaming failed when loading .resource file. Origin: %s, 
Target: %s, because %s",
-              resourceFileToLoad.getAbsolutePath(),
-              targetResourceFile.getAbsolutePath(),
-              e.getMessage()));
-    }
 
-    final File modFileToLoad =
-        fsFactory.getFile(tsFileToLoad.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX);
-    try {
-      if (modFileToLoad.exists()) {
-        // when successfully loaded, the filepath of the resource will be 
changed to the IoTDB data
-        // dir, so we can add a suffix to find the old modification file.
-        final File targetModFile =
-            fsFactory.getFile(targetFile.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX);
-        try {
-          RetryUtils.retryOnException(
-              () -> {
-                Files.deleteIfExists(targetModFile.toPath());
-                return null;
-              });
-        } catch (final IOException e) {
-          logger.warn("Cannot delete localModFile {}", targetModFile, e);
-        }
+      // Keep all source files intact until the complete destination is 
published.
+      loadModFile(tsFileToLoad, targetFile);
+      try {
+        Files.move(pendingFile.toPath(), targetFile.toPath(), 
StandardCopyOption.ATOMIC_MOVE);
+      } catch (IOException e) {
+        throw new LoadFileException(e);
+      }
+      loadPublished = true;
+    } finally {
+      tsFileResource.setModFile(null);
+      if (!loadPublished && pendingFile.exists()) {
         try {
-          final long modFileSize = modFileToLoad.length();
-          if (deleteOriginFile) {
-            RetryUtils.retryOnException(
-                () -> {
-                  FileUtils.moveFile(modFileToLoad, targetModFile);
-                  return null;
-                });
-          } else {
-            RetryUtils.retryOnException(
-                () -> {
-                  Files.copy(modFileToLoad.toPath(), targetModFile.toPath());
-                  return null;
-                });
-          }
-
-          FileMetrics.getInstance().increaseModFileNum(1);
-          FileMetrics.getInstance().increaseModFileSize(modFileSize);
-        } catch (final IOException e) {
+          deleteUnpublishedLoad(pendingFile);
+        } catch (IOException e) {
           logger.warn(
-              "File renaming failed when loading .mod file. Origin: {}, 
Target: {}",
-              modFileToLoad.getAbsolutePath(),
-              targetModFile.getAbsolutePath(),
+              
StorageEngineMessages.LOG_FAILED_TO_DELETE_FILE_OR_DIRECTORY_ARG_7EC21473,
+              pendingFile,
               e);
-          throw new LoadFileException(
-              String.format(
-                  "File renaming failed when loading .mod file. Origin: %s, 
Target: %s, because %s",
-                  modFileToLoad.getAbsolutePath(),
-                  targetModFile.getAbsolutePath(),
-                  e.getMessage()));
         }
       }
-    } finally {
-      // ModFile will be updated during the next call to `getModFile`
-      tsFileResource.setModFile(null);
+    }
+
+    // Only published mods belong in FileMetrics. Also replace any cached 
source-side handle.
+    tsFileResource.setModFile(ModificationFile.getNormalMods(tsFileResource));
+    if (tsFileResource.modFileExists()) {
+      FileMetrics.getInstance().increaseModFileNum(1);
+      
FileMetrics.getInstance().increaseModFileSize(tsFileResource.getModFile().getSize());
     }
 
     // Listen before the tsFile is added into tsFile manager to avoid it being 
compacted
@@ -3653,9 +3654,74 @@ public class DataRegion implements IDataRegionForQuery {
 
     tsFileManager.add(tsFileResource, false);
 
+    if (deleteOriginFile) {
+      for (File sourceFile :
+          new File[] {
+            tsFileToLoad, resourceFileToLoad, new File(tsFileToLoad + 
ModificationFile.FILE_SUFFIX)
+          }) {
+        try {
+          RetryUtils.retryOnException(
+              () -> {
+                Files.deleteIfExists(sourceFile.toPath());
+                return null;
+              });
+        } catch (IOException e) {
+          // The installed TsFile is already visible and registered; source 
cleanup cannot roll it
+          // back.
+          logger.warn(
+              
StorageEngineMessages.LOG_FAILED_TO_DELETE_FILE_OR_DIRECTORY_ARG_7EC21473,
+              sourceFile,
+              e);
+          break;
+        }
+      }
+    }
+
     return true;
   }
 
+  private void loadModFile(File tsFileToLoad, File targetFile) throws 
LoadFileException {
+    final File modFileToLoad =
+        fsFactory.getFile(tsFileToLoad.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX);
+    if (modFileToLoad.exists()) {
+      // when successfully loaded, the filepath of the resource will be 
changed to the IoTDB data
+      // dir, so we can add a suffix to find the old modification file.
+      final File targetModFile =
+          fsFactory.getFile(targetFile.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX);
+      try {
+        RetryUtils.retryOnException(
+            () -> {
+              Files.deleteIfExists(targetModFile.toPath());
+              return null;
+            });
+      } catch (final IOException e) {
+        logger.warn(
+            StorageEngineMessages.LOG_CANNOT_DELETE_LOCALMODFILE_ARG_F9CD0EA6, 
targetModFile, e);
+      }
+      try {
+        RetryUtils.retryOnException(
+            () -> {
+              Files.copy(modFileToLoad.toPath(), targetModFile.toPath());
+              return null;
+            });
+      } catch (final IOException e) {
+        logger.warn(
+            StorageEngineMessages
+                
.LOG_FILE_RENAMING_FAILED_WHEN_LOADING_MOD_FILE_ORIGIN_ARG_TARGET_ARG_2493B8EE,
+            modFileToLoad.getAbsolutePath(),
+            targetModFile.getAbsolutePath(),
+            e);
+        throw new LoadFileException(
+            String.format(
+                StorageEngineMessages
+                    
.EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_MOD_FILE_ORIGIN_ARG_TARGET_ARG_BECAUSE_ARG_93A853C4,
+                modFileToLoad.getAbsolutePath(),
+                targetModFile.getAbsolutePath(),
+                e.getMessage()));
+      }
+    }
+  }
+
   /**
    * Get all working sequence tsfile processors
    *
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java
index 9dc760e9a65..3edb49f1daf 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java
@@ -90,11 +90,11 @@ public class LoadTsFileNodeTest {
   }
 
   @Test
-  public void testCleanContinuesAfterOneFileCannotBeDeleted() throws Exception 
{
+  public void testCleanRetainsSidecarsWhenTsFileCannotBeDeleted() throws 
Exception {
     final File tempDir = Files.createTempDirectory("load-node-clean").toFile();
     try {
-      // A non-empty directory at the TsFile path makes that deletion fail 
deterministically. The
-      // companion cleanup must still continue instead of sharing the same 
try-catch block.
+      // A retained source TsFile must keep its resource and deletions for any 
later explicit LOAD.
+      // A non-empty directory at the TsFile path makes that deletion fail 
deterministically.
       final File tsFile = new File(tempDir, "1-0-0-0.tsfile");
       Assert.assertTrue(tsFile.mkdirs());
       Assert.assertTrue(new File(tsFile, "non-empty").createNewFile());
@@ -108,7 +108,33 @@ public class LoadTsFileNodeTest {
       node.clean();
 
       Assert.assertTrue(tsFile.exists());
-      Assert.assertFalse(resourceFile.exists());
+      Assert.assertTrue(resourceFile.isFile());
+      Assert.assertTrue(modsFile.isFile());
+    } finally {
+      deleteRecursively(tempDir);
+    }
+  }
+
+  @Test
+  public void testCleanContinuesAfterOneSidecarCannotBeDeleted() throws 
Exception {
+    final File tempDir = Files.createTempDirectory("load-node-clean").toFile();
+    try {
+      final File tsFile = new File(tempDir, "1-0-0-0.tsfile");
+      final File resourceFile = new File(tsFile + 
TsFileResource.RESOURCE_SUFFIX);
+      final File modsFile = new File(tsFile + ModificationFile.FILE_SUFFIX);
+      Assert.assertTrue(tsFile.createNewFile());
+      Assert.assertTrue(resourceFile.mkdir());
+      final File retained = new File(resourceFile, "non-empty");
+      Assert.assertTrue(retained.createNewFile());
+      Assert.assertTrue(modsFile.createNewFile());
+
+      final LoadSingleTsFileNode node =
+          new LoadSingleTsFileNode(new PlanNodeId(""), new 
TsFileResource(tsFile), true, 0L);
+      node.clean();
+
+      Assert.assertFalse(tsFile.exists());
+      Assert.assertTrue(resourceFile.isDirectory());
+      Assert.assertTrue(retained.isFile());
       Assert.assertFalse(modsFile.exists());
     } finally {
       deleteRecursively(tempDir);
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
index 93223cc8ccf..ce3973c23f8 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
@@ -37,6 +37,7 @@ import org.apache.iotdb.db.exception.DataRegionException;
 import org.apache.iotdb.db.exception.TsFileProcessorException;
 import org.apache.iotdb.db.exception.WriteProcessException;
 import org.apache.iotdb.db.exception.WriteProcessRejectException;
+import org.apache.iotdb.db.exception.load.LoadFileException;
 import org.apache.iotdb.db.exception.query.QueryProcessException;
 import org.apache.iotdb.db.queryengine.common.QueryId;
 import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
@@ -61,6 +62,8 @@ import 
org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy;
 import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
 import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk;
 import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
 import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
@@ -194,6 +197,147 @@ public class DataRegionTest {
     return insertRowNode;
   }
 
+  @Test
+  public void testLoadInstallationRetainsOrDeletesTheCompleteSourceGroup() 
throws Exception {
+    for (boolean deleteOrigin : new boolean[] {false, true}) {
+      for (boolean withMods : new boolean[] {false, true}) {
+        final File sourceDir =
+            new File(TestConstant.OUTPUT_DATA_DIR, "load-source-" + 
deleteOrigin + "-" + withMods);
+        Files.createDirectories(sourceDir.toPath());
+        final File source = new File(sourceDir, "1-1-0-0.tsfile");
+        final MeasurementSchema schema = new MeasurementSchema(measurementId, 
TSDataType.INT32);
+        try (TsFileWriter writer = new TsFileWriter(source)) {
+          writer.registerAlignedTimeseries(new Path(deviceId), 
Collections.singletonList(schema));
+          final Tablet tablet = new Tablet(deviceId, 
Collections.singletonList(schema), 3);
+          for (int row = 0; row < 3; row++) {
+            tablet.addTimestamp(row, row + 1L);
+            tablet.addValue(measurementId, row, row);
+          }
+          tablet.rowSize = 3;
+          writer.writeAligned(tablet);
+        }
+        final TsFileResource resource = Mockito.spy(new 
TsFileResource(source));
+        resource.setStatus(TsFileResourceStatus.NORMAL);
+        resource.updateStartTime(new PlainDeviceID(deviceId), 1);
+        resource.updateEndTime(new PlainDeviceID(deviceId), 3);
+        resource.serialize();
+        final File sourceMods = new File(source + 
ModificationFile.FILE_SUFFIX);
+        if (withMods) {
+          try (ModificationFile mods = resource.getModFile()) {
+            mods.write(
+                new Deletion(
+                    new MeasurementPath(deviceId + "." + measurementId), 
Long.MAX_VALUE, 2, 2));
+          }
+        }
+        // Exercise rebinding even when the source handle was cached before 
installation.
+        resource.getModFile();
+        final byte[] data = Files.readAllBytes(source.toPath());
+        final byte[] deletions = withMods ? 
Files.readAllBytes(sourceMods.toPath()) : null;
+        final File sourceResource = new File(source + 
TsFileResource.RESOURCE_SUFFIX);
+        final File savedResource = new File(sourceDir, "saved-resource");
+        Files.move(sourceResource.toPath(), savedResource.toPath());
+        try {
+          final LoadFileException failure =
+              Assert.assertThrows(
+                  LoadFileException.class,
+                  () -> dataRegion.loadNewTsFile(resource, deleteOrigin, 
false, true));
+          Assert.assertTrue(
+              failure.getMessage(),
+              failure.getMessage().contains(sourceResource.getAbsolutePath()));
+          final File rejectedTarget = resource.getTsFile();
+          Assert.assertNotEquals(source.getAbsolutePath(), 
rejectedTarget.getAbsolutePath());
+          Assert.assertFalse(rejectedTarget.exists());
+          Assert.assertFalse(new File(rejectedTarget + 
".load-pending").exists());
+          Assert.assertFalse(new File(rejectedTarget + 
TsFileResource.RESOURCE_SUFFIX).exists());
+          Assert.assertFalse(new File(rejectedTarget + 
ModificationFile.FILE_SUFFIX).exists());
+          Assert.assertArrayEquals(data, Files.readAllBytes(source.toPath()));
+          Assert.assertEquals(withMods, sourceMods.exists());
+        } finally {
+          Files.move(savedResource.toPath(), sourceResource.toPath());
+          resource.setFile(source);
+        }
+        final int modFilesBeforeLoad = 
FileMetrics.getInstance().getModFileNum();
+        final long modSizeBeforeLoad = getModFileSize();
+        if (withMods) {
+          // A nonempty destination directory rejects atomic publication on 
Windows and Unix.
+          Mockito.doAnswer(
+                  invocation -> {
+                    invocation.callRealMethod();
+                    final File file = invocation.getArgument(0);
+                    if (!file.getParentFile()
+                        .getCanonicalFile()
+                        .equals(sourceDir.getCanonicalFile())) {
+                      Files.createDirectories(file.toPath());
+                      Files.write(file.toPath().resolve("block-publication"), 
new byte[] {1});
+                    }
+                    return null;
+                  })
+              .when(resource)
+              .setFile(Mockito.any(File.class));
+          File rejectedTarget = null;
+          try {
+            Assert.assertThrows(
+                LoadFileException.class,
+                () -> dataRegion.loadNewTsFile(resource, deleteOrigin, false, 
true));
+            rejectedTarget = resource.getTsFile();
+            Assert.assertTrue(rejectedTarget.isDirectory());
+            Assert.assertTrue(new File(rejectedTarget + 
TsFileResource.RESOURCE_SUFFIX).isFile());
+            Assert.assertTrue(new File(rejectedTarget + 
ModificationFile.FILE_SUFFIX).isFile());
+            Assert.assertFalse(new File(rejectedTarget + 
".load-pending").exists());
+            Assert.assertEquals(modFilesBeforeLoad, 
FileMetrics.getInstance().getModFileNum());
+            Assert.assertEquals(modSizeBeforeLoad, getModFileSize());
+            Assert.assertArrayEquals(data, 
Files.readAllBytes(source.toPath()));
+            Assert.assertArrayEquals(deletions, 
Files.readAllBytes(sourceMods.toPath()));
+            Assert.assertTrue(sourceResource.isFile());
+          } finally {
+            
Mockito.doCallRealMethod().when(resource).setFile(Mockito.any(File.class));
+            if (rejectedTarget != null) {
+              // Rollback preserves sidecars at an occupied target; remove 
this test's
+              // blocker/group.
+              
Files.delete(rejectedTarget.toPath().resolve("block-publication"));
+              Files.delete(rejectedTarget.toPath());
+              Files.deleteIfExists(
+                  new File(rejectedTarget + 
TsFileResource.RESOURCE_SUFFIX).toPath());
+              Files.deleteIfExists(
+                  new File(rejectedTarget + 
ModificationFile.FILE_SUFFIX).toPath());
+            }
+            resource.setFile(source);
+          }
+        }
+        dataRegion.loadNewTsFile(resource, deleteOrigin, false, true);
+        final File installed = resource.getTsFile();
+        Assert.assertArrayEquals(data, Files.readAllBytes(installed.toPath()));
+        Assert.assertTrue(new File(installed + 
TsFileResource.RESOURCE_SUFFIX).isFile());
+        Assert.assertFalse(new File(installed + ".load-pending").exists());
+        final File installedMods = new File(installed + 
ModificationFile.FILE_SUFFIX);
+        Assert.assertEquals(
+            installedMods.getCanonicalFile(),
+            new File(resource.getModFile().getFilePath()).getCanonicalFile());
+        Assert.assertEquals(withMods ? 1 : 0, 
resource.getModFile().getModifications().size());
+        Assert.assertEquals(withMods, installedMods.exists());
+        Assert.assertEquals(
+            modFilesBeforeLoad + (withMods ? 1 : 0), 
FileMetrics.getInstance().getModFileNum());
+        Assert.assertEquals(modSizeBeforeLoad + installedMods.length(), 
getModFileSize());
+        if (withMods) {
+          Assert.assertArrayEquals(deletions, 
Files.readAllBytes(installedMods.toPath()));
+        }
+        Assert.assertEquals(!deleteOrigin, source.exists());
+        Assert.assertEquals(
+            !deleteOrigin, new File(source + 
TsFileResource.RESOURCE_SUFFIX).exists());
+        Assert.assertEquals(withMods && !deleteOrigin, sourceMods.exists());
+      }
+    }
+  }
+
+  private static long getModFileSize() throws Exception {
+    final Field metricsField = 
FileMetrics.class.getDeclaredField("MODS_FILE_METRICS");
+    metricsField.setAccessible(true);
+    final Object metrics = metricsField.get(null);
+    final Field sizeField = metrics.getClass().getDeclaredField("modFileSize");
+    sizeField.setAccessible(true);
+    return ((java.util.concurrent.atomic.AtomicLong) 
sizeField.get(metrics)).get();
+  }
+
   @Test
   public void testLoadRateLimiterAccountsForSourceFileBeforeTransfer() throws 
Exception {
     final LoadTsFileCostMetricsSet metrics = 
LoadTsFileCostMetricsSet.getInstance();

Reply via email to