This is an automated email from the ASF dual-hosted git repository.

Caideyipi pushed a commit to branch codex/load-local-install-crash-safety
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 100718bdc589886bfda75205045952c51dfcf88e
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 5 13:37:49 2026 +0800

    Prevent partial local LOAD installation after process crashes
---
 .../iotdb/it/env/cluster/config/MppJVMConfig.java  |  11 +
 .../it/env/cluster/node/AbstractNodeWrapper.java   |   5 +
 .../iotdb/db/it/IoTDBLoadTsFileInstallIT.java      | 366 +++++++++++++++++++++
 .../org/apache/iotdb/db/it/utils/NodeDebugger.java | 232 +++++++++++++
 .../db/storageengine/dataregion/DataRegion.java    | 167 ++++++----
 .../storageengine/dataregion/DataRegionTest.java   |  85 +++++
 6 files changed, 811 insertions(+), 55 deletions(-)

diff --git 
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
 
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
index d9c51717cf8..7e46af79736 100644
--- 
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
+++ 
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
@@ -29,6 +29,16 @@ public class MppJVMConfig implements JVMConfig {
   private int maxHeapSize;
   private int maxDirectMemorySize;
   private String timezone;
+  private int debugPort;
+
+  /** Enables loopback-only JDWP for tests that need to observe an exact node 
execution stage. */
+  public void setDebugPort(int debugPort) {
+    this.debugPort = debugPort;
+  }
+
+  public int getDebugPort() {
+    return debugPort;
+  }
 
   @Override
   public JVMConfig setInitHeapSize(int initSize) {
@@ -94,6 +104,7 @@ public class MppJVMConfig implements JVMConfig {
     this.setMaxHeapSize(config.getMaxHeapSize());
     this.setMaxDirectMemorySize(config.getMaxDirectMemorySize());
     this.setTimezone(config.getTimezone());
+    this.setDebugPort(config.getDebugPort());
   }
 
   public static Builder builder() {
diff --git 
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
 
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
index 8d3f5f2cc9d..3e6d7827eb2 100644
--- 
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
+++ 
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
@@ -483,6 +483,11 @@ public abstract class AbstractNodeWrapper implements 
BaseNodeWrapper {
       File stdoutFile = new File(getLogPath());
       List<String> startCmd = new ArrayList<>();
       startCmd.add(JAVA_CMD);
+      if (jvmConfig.getDebugPort() > 0) {
+        startCmd.add(
+            
"-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=127.0.0.1:"
+                + jvmConfig.getDebugPort());
+      }
       if (!SystemUtils.IS_JAVA_1_8) {
         startCmd.add("--add-opens=java.base/java.util.concurrent=ALL-UNNAMED");
         startCmd.add("--add-opens=java.base/java.lang=ALL-UNNAMED");
diff --git 
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileInstallIT.java
 
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileInstallIT.java
new file mode 100644
index 00000000000..df10d34f803
--- /dev/null
+++ 
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileInstallIT.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.it.utils.NodeDebugger;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
+import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.config.MppJVMConfig;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+
+import com.sun.jdi.ObjectReference;
+import com.sun.jdi.StackFrame;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.awaitility.Awaitility;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.net.ServerSocket;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/** A single-replica process fixture for the local whole-file installation 
commit boundary. */
+@RunWith(IoTDBTestRunner.class)
+@Category(ClusterIT.class)
+public class IoTDBLoadTsFileInstallIT {
+  private static final String DATABASE = "root.load.install";
+  private static final String DATA_REGION =
+      "org.apache.iotdb.db.storageengine.dataregion.DataRegion";
+  private static final long[] TIMES = {-3, -2, -1};
+  private static final List<IMeasurementSchema> SCHEMAS =
+      Arrays.asList(
+          new MeasurementSchema("s", TSDataType.INT64),
+          new MeasurementSchema("txt", TSDataType.TEXT));
+  private static int debugPort;
+  private File directory;
+
+  @BeforeClass
+  public static void setUpClass() throws Exception {
+    try (ServerSocket socket = new ServerSocket(0)) {
+      debugPort = socket.getLocalPort();
+    }
+    ((MppJVMConfig) 
EnvFactory.getEnv().getConfig().getDataNodeJVMConfig()).setDebugPort(debugPort);
+    EnvFactory.getEnv()
+        .getConfig()
+        .getCommonConfig()
+        .setDataReplicationFactor(1)
+        .setSchemaReplicationFactor(1)
+        .setDataRegionConsensusProtocolClass(ConsensusFactory.SIMPLE_CONSENSUS)
+        .setDefaultDataRegionGroupNumPerDatabase(1)
+        .setDataRegionGroupExtensionPolicy("CUSTOM");
+    EnvFactory.getEnv().initClusterEnvironment(1, 1);
+  }
+
+  @AfterClass
+  public static void tearDownClass() {
+    try {
+      EnvFactory.getEnv().cleanClusterEnvironment();
+    } finally {
+      ((MppJVMConfig) 
EnvFactory.getEnv().getConfig().getDataNodeJVMConfig()).setDebugPort(0);
+    }
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    directory = Files.createTempDirectory("load-install-").toFile();
+    execute("create database " + DATABASE);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    try {
+      if (!node().isAlive()) {
+        startNode();
+      }
+      execute("delete database " + DATABASE);
+    } finally {
+      try (Stream<Path> paths = Files.walk(directory.toPath())) {
+        for (Path path : 
paths.sorted(Comparator.reverseOrder()).collect(Collectors.toList())) {
+          Files.delete(path);
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testCrashBeforeDeletionInstallationDoesNotPublishPartialFile() 
throws Exception {
+    assertCrashBoundary(false, false);
+  }
+
+  @Test
+  public void testCrashAfterDeletionInstallationDoesNotPublishPartialFile() 
throws Exception {
+    assertCrashBoundary(false, true);
+  }
+
+  @Test
+  public void testPublishedFileSurvivesCrashBeforeClientAcknowledgement() 
throws Exception {
+    assertCrashBoundary(true, true);
+  }
+
+  private void assertCrashBoundary(boolean published, boolean 
deletionsInstalled) throws Exception {
+    for (boolean delete : new boolean[] {false, true}) {
+      for (boolean legacy : new boolean[] {false, true}) {
+        final String prefix = DATABASE + ".d" + delete + "_v" + legacy;
+        final File source =
+            new File(directory, (100 + (delete ? 2 : 0) + (legacy ? 1 : 0)) + 
"-0-0-0.tsfile");
+        writeSource(source, prefix, legacy);
+        final ExecutorService executor = Executors.newSingleThreadExecutor();
+        final File target;
+        final boolean visibleBeforeCrash;
+        final Future<?> request;
+        try {
+          try (NodeDebugger debugger =
+              new NodeDebugger(
+                  debugPort, DATA_REGION, published ? "loadTsFileToUnSequence" 
: "loadModFile")) {
+            request =
+                executor.submit(
+                    () -> {
+                      execute(load(source, delete));
+                      return null;
+                    });
+            StackFrame frame =
+                debugger.await(
+                    candidate ->
+                        
NodeDebugger.filePath(candidate.getArgumentValues().get(0))
+                            .equals(source.getAbsolutePath()));
+            if (published) {
+              debugger.pauseOnExit();
+              debugger.resume();
+              frame = debugger.await(candidate -> true);
+              final ObjectReference resource = (ObjectReference) 
frame.getArgumentValues().get(1);
+              target =
+                  new File(
+                      NodeDebugger.filePath(
+                          
resource.getValue(resource.referenceType().fieldByName("file"))));
+              Assert.assertTrue(
+                  "Published file must already include all deletions",
+                  ModificationFile.getExclusiveMods(target).isFile());
+            } else {
+              target = new 
File(NodeDebugger.filePath(frame.getArgumentValues().get(1)));
+              if (deletionsInstalled) {
+                debugger.pauseOnExit();
+                debugger.resume();
+                debugger.await(candidate -> true);
+              }
+              Assert.assertEquals(
+                  deletionsInstalled, 
ModificationFile.getExclusiveMods(target).isFile());
+            }
+            Assert.assertTrue(new File(target + 
TsFileResource.RESOURCE_SUFFIX).isFile());
+            visibleBeforeCrash = target.isFile();
+            Assert.assertFalse(
+                "The original SQL must still be awaiting its response", 
request.isDone());
+            node().stopForcibly();
+            final ExecutionException failure =
+                Assert.assertThrows(
+                    ExecutionException.class, () -> request.get(30, 
TimeUnit.SECONDS));
+            Assert.assertTrue(failure.toString(), failure.getCause() 
instanceof SQLException);
+          }
+          // The commit can complete its source cleanup before the client 
receives a response.
+          assertSource(source, legacy, !published || !delete);
+          startNode();
+          if (published) {
+            Assert.assertTrue(visibleBeforeCrash);
+            assertRows(prefix);
+          } else {
+            // Check recovery before any new LOAD; retrying first could 
conceal resurrection.
+            try (Connection connection = EnvFactory.getEnv().getConnection();
+                Statement statement = connection.createStatement();
+                ResultSet rows =
+                    statement.executeQuery("select s from " + prefix + ".d0 
where time=-2")) {
+              Assert.assertFalse("Recovery must not resurrect a deleted 
value", rows.next());
+            }
+            Assert.assertFalse(
+                "A file without its deletions must never be published", 
visibleBeforeCrash);
+            try (Connection connection = EnvFactory.getEnv().getConnection();
+                Statement statement = connection.createStatement();
+                ResultSet rows = statement.executeQuery("select ** from " + 
prefix + ".**")) {
+              Assert.assertFalse(
+                  "Unpublished data must stay invisible after recovery", 
rows.next());
+            }
+            Assert.assertFalse(target.exists());
+            Assert.assertFalse(new File(target + ".load-pending").exists());
+            Assert.assertFalse(new File(target + 
TsFileResource.RESOURCE_SUFFIX).exists());
+            
Assert.assertFalse(ModificationFile.getExclusiveMods(target).exists());
+            
Assert.assertFalse(ModificationFileV1.getNormalMods(target).exists());
+            // This is an explicit new submission, not recovery of the 
interrupted request.
+            execute(load(source, delete));
+            assertRows(prefix);
+            assertSource(source, legacy, !delete);
+          }
+        } finally {
+          executor.shutdownNow();
+          Assert.assertTrue(executor.awaitTermination(30, TimeUnit.SECONDS));
+        }
+      }
+    }
+  }
+
+  private static void writeSource(File source, String prefix, boolean legacy) 
throws Exception {
+    try (TsFileWriter writer = new TsFileWriter(source)) {
+      for (int device = 0; device < 2; device++) {
+        final String path = prefix + ".d" + device;
+        writer.registerAlignedTimeseries(path, SCHEMAS);
+        final Tablet tablet = new Tablet(path, SCHEMAS, TIMES.length);
+        for (int row = 0; row < TIMES.length; row++) {
+          tablet.addTimestamp(row, TIMES[row]);
+          tablet.addValue(row, "s", device * 100L + row);
+          tablet.addValue(row, "txt", row == 0 ? null : "d" + device + "_r" + 
row);
+        }
+        writer.writeAligned(tablet);
+      }
+    }
+    final TsFileResource resource = new TsFileResource(source);
+    for (int device = 0; device < 2; device++) {
+      final IDeviceID id = IDeviceID.Factory.DEFAULT_FACTORY.create(prefix + 
".d" + device);
+      resource.updateStartTime(id, TIMES[0]);
+      resource.updateEndTime(id, TIMES[TIMES.length - 1]);
+    }
+    resource.serialize();
+    if (legacy) {
+      try (ModificationFileV1 mods = 
ModificationFileV1.getNormalMods(resource)) {
+        for (int device = 0; device < 2; device++) {
+          mods.write(
+              new Deletion(
+                  new MeasurementPath(prefix + ".d" + device + ".s"),
+                  Long.MAX_VALUE,
+                  TIMES[1],
+                  TIMES[1]));
+        }
+      }
+    } else {
+      try (ModificationFile mods = resource.getExclusiveModFile()) {
+        for (int device = 0; device < 2; device++) {
+          mods.write(
+              new TreeDeletionEntry(
+                  new MeasurementPath(prefix + ".d" + device + ".s"), 
TIMES[1], TIMES[1]));
+        }
+      }
+    }
+  }
+
+  private static void assertRows(String prefix) throws Exception {
+    for (int device = 0; device < 2; device++) {
+      try (Connection connection = EnvFactory.getEnv().getConnection();
+          Statement statement = connection.createStatement();
+          ResultSet rows = statement.executeQuery("select s,txt from " + 
prefix + ".d" + device)) {
+        for (int row = 0; row < TIMES.length; row++) {
+          Assert.assertTrue(rows.next());
+          Assert.assertEquals(TIMES[row], rows.getLong(1));
+          Assert.assertEquals(row == 1 ? null : device * 100L + row, 
rows.getObject(2));
+          Assert.assertEquals(row == 0 ? null : "d" + device + "_r" + row, 
rows.getString(3));
+        }
+        Assert.assertFalse(rows.next());
+      }
+    }
+  }
+
+  private static void assertSource(File source, boolean legacy, boolean 
exists) {
+    Assert.assertEquals(exists, source.isFile());
+    Assert.assertEquals(exists, new File(source + 
TsFileResource.RESOURCE_SUFFIX).isFile());
+    Assert.assertEquals(
+        exists,
+        (legacy
+                ? ModificationFileV1.getNormalMods(source)
+                : ModificationFile.getExclusiveMods(source))
+            .isFile());
+  }
+
+  private static String load(File source, boolean delete) {
+    return "load '"
+        + source.getAbsolutePath()
+        + "' with 
('database-level'='2','tablet-conversion-threshold'='-1','on-success'='"
+        + (delete ? "delete" : "none")
+        + "')";
+  }
+
+  private static void execute(String sql) throws Exception {
+    try (Connection connection =
+            
EnvFactory.getEnv().getWriteOnlyConnectionWithSpecifiedDataNode(node());
+        Statement statement = connection.createStatement()) {
+      statement.execute(sql);
+    }
+  }
+
+  private static DataNodeWrapper node() {
+    return EnvFactory.getEnv().getDataNodeWrapperList().get(0);
+  }
+
+  private static void startNode() throws Exception {
+    node().start();
+    EnvFactory.getEnv()
+        .ensureNodeStatus(
+            Collections.singletonList(node()), 
Collections.singletonList(NodeStatus.Running));
+    Awaitility.await()
+        .atMost(60, TimeUnit.SECONDS)
+        .untilAsserted(
+            () -> {
+              try (Connection connection = EnvFactory.getEnv().getConnection();
+                  Statement statement = connection.createStatement();
+                  ResultSet regions = statement.executeQuery("show regions")) {
+                int count = 0;
+                while (regions.next()) {
+                  if (DATABASE.equals(regions.getString("Database"))
+                      && "DataRegion".equals(regions.getString("Type"))) {
+                    count++;
+                    Assert.assertEquals(
+                        NodeStatus.Running.getStatus(), 
regions.getString("Status"));
+                  }
+                }
+                Assert.assertTrue("The actual DataRegion must be ready after 
restart", count > 0);
+              }
+            });
+  }
+}
diff --git 
a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/NodeDebugger.java 
b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/NodeDebugger.java
new file mode 100644
index 00000000000..c37bd06f1a1
--- /dev/null
+++ 
b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/NodeDebugger.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it.utils;
+
+import com.sun.jdi.Bootstrap;
+import com.sun.jdi.ObjectReference;
+import com.sun.jdi.ReferenceType;
+import com.sun.jdi.StackFrame;
+import com.sun.jdi.StringReference;
+import com.sun.jdi.ThreadReference;
+import com.sun.jdi.VMDisconnectedException;
+import com.sun.jdi.Value;
+import com.sun.jdi.VirtualMachine;
+import com.sun.jdi.connect.AttachingConnector;
+import com.sun.jdi.connect.Connector;
+import com.sun.jdi.event.BreakpointEvent;
+import com.sun.jdi.event.ClassPrepareEvent;
+import com.sun.jdi.event.Event;
+import com.sun.jdi.event.EventSet;
+import com.sun.jdi.event.MethodEntryEvent;
+import com.sun.jdi.event.MethodExitEvent;
+import com.sun.jdi.event.VMStartEvent;
+import com.sun.jdi.request.BreakpointRequest;
+import com.sun.jdi.request.ClassPrepareRequest;
+import com.sun.jdi.request.EventRequest;
+import com.sun.jdi.request.MethodEntryRequest;
+import com.sun.jdi.request.MethodExitRequest;
+import org.junit.Assert;
+
+import java.net.ConnectException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+/**
+ * Pauses only the node thread executing a selected method. Other threads 
(including the active
+ * scanner) keep running. All control stays in the test JVM; no product hooks 
or agent bytecode are
+ * installed. Closing the debugger releases a paused thread even when an 
assertion fails.
+ */
+public final class NodeDebugger implements AutoCloseable {
+  private final VirtualMachine vm;
+  private final String className;
+  private final String methodName;
+  private final boolean observeMethodEntries;
+  private final Set<ReferenceType> breakpointTypes = new HashSet<>();
+  private EventSet suspended;
+  private ThreadReference pausedThread;
+
+  public NodeDebugger(int port, String className, String methodName) throws 
Exception {
+    this(port, className, methodName, false);
+  }
+
+  public NodeDebugger(int port, String className, String methodName, boolean 
observeMethodEntries)
+      throws Exception {
+    this.className = className;
+    this.methodName = methodName;
+    this.observeMethodEntries = observeMethodEntries;
+    final AttachingConnector connector =
+        Bootstrap.virtualMachineManager().attachingConnectors().stream()
+            .filter(candidate -> 
candidate.name().equals("com.sun.jdi.SocketAttach"))
+            .findFirst()
+            .orElseThrow(AssertionError::new);
+    final Map<String, Connector.Argument> arguments = 
connector.defaultArguments();
+    arguments.get("hostname").setValue("127.0.0.1");
+    arguments.get("port").setValue(Integer.toString(port));
+    arguments.get("timeout").setValue("10000");
+    final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+    VirtualMachine connected;
+    while (true) {
+      try {
+        connected = connector.attach(arguments);
+        break;
+      } catch (ConnectException e) {
+        if (System.nanoTime() >= deadline) {
+          throw e;
+        }
+        // ProcessBuilder.start returns before the suspended VM has opened its 
JDWP listener.
+        TimeUnit.MILLISECONDS.sleep(100);
+      }
+    }
+    vm = connected;
+    try {
+      if (observeMethodEntries) {
+        installMethodEntryRequest();
+      } else {
+        final ClassPrepareRequest prepare = 
vm.eventRequestManager().createClassPrepareRequest();
+        prepare.addClassFilter(className);
+        prepare.setSuspendPolicy(EventRequest.SUSPEND_EVENT_THREAD);
+        prepare.enable();
+        for (ReferenceType type : vm.classesByName(className)) {
+          if (type.isPrepared()) {
+            installBreakpoint(type);
+          }
+        }
+      }
+    } catch (Exception | AssertionError e) {
+      vm.dispose();
+      throw e;
+    }
+  }
+
+  private void installMethodEntryRequest() {
+    final MethodEntryRequest entry = 
vm.eventRequestManager().createMethodEntryRequest();
+    entry.addClassFilter(className);
+    entry.setSuspendPolicy(EventRequest.SUSPEND_EVENT_THREAD);
+    entry.enable();
+  }
+
+  /** Releases only the initial VMStart suspension after all requests have 
been installed. */
+  public void resumeStartup() throws Exception {
+    suspended = vm.eventQueue().remove(10000);
+    Assert.assertNotNull("Suspended startup must produce a VMStart event", 
suspended);
+    Assert.assertTrue(suspended.stream().anyMatch(event -> event instanceof 
VMStartEvent));
+    resume();
+  }
+
+  private void installBreakpoint(ReferenceType type) {
+    if (!breakpointTypes.add(type)) {
+      return;
+    }
+    Assert.assertEquals(className + "." + methodName, 1, 
type.methodsByName(methodName).size());
+    final BreakpointRequest breakpoint =
+        vm.eventRequestManager()
+            
.createBreakpointRequest(type.methodsByName(methodName).get(0).location());
+    breakpoint.setSuspendPolicy(EventRequest.SUSPEND_EVENT_THREAD);
+    breakpoint.enable();
+  }
+
+  public StackFrame await(Predicate<StackFrame> condition) throws Exception {
+    final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60);
+    while (System.nanoTime() < deadline) {
+      final EventSet events =
+          vm.eventQueue()
+              .remove(Math.max(1, TimeUnit.NANOSECONDS.toMillis(deadline - 
System.nanoTime())));
+      if (events == null) {
+        break;
+      }
+      // Remember ownership before inspecting a frame: failed predicates must 
also release it.
+      suspended = events;
+      for (Event event : events) {
+        if (event instanceof ClassPrepareEvent && !observeMethodEntries) {
+          installBreakpoint(((ClassPrepareEvent) event).referenceType());
+        } else if (event instanceof MethodEntryEvent
+            && ((MethodEntryEvent) event).method().name().equals(methodName)) {
+          final ThreadReference thread = ((MethodEntryEvent) event).thread();
+          final StackFrame frame = thread.frame(0);
+          if (condition.test(frame)) {
+            pausedThread = thread;
+            return frame;
+          }
+        } else if (event instanceof BreakpointEvent) {
+          final StackFrame frame = ((BreakpointEvent) event).thread().frame(0);
+          if (condition.test(frame)) {
+            pausedThread = ((BreakpointEvent) event).thread();
+            return frame;
+          }
+        } else if (event instanceof MethodExitEvent
+            && ((MethodExitEvent) event).method().name().equals(methodName)) {
+          final StackFrame frame = ((MethodExitEvent) event).thread().frame(0);
+          if (condition.test(frame)) {
+            // Resume must let cleanup and the SQL response run without 
another exit pause.
+            event.request().disable();
+            return frame;
+          }
+        }
+      }
+      resume();
+    }
+    throw new AssertionError("Did not reach " + className + "." + methodName);
+  }
+
+  public void resume() {
+    if (suspended != null) {
+      suspended.resume();
+      suspended = null;
+    }
+  }
+
+  /** Changes an entry pause into an exit pause on the same method and 
connection. */
+  public void pauseOnExit() {
+    vm.eventRequestManager()
+        .deleteEventRequests(
+            new 
java.util.ArrayList<>(vm.eventRequestManager().breakpointRequests()));
+    vm.eventRequestManager()
+        .deleteEventRequests(
+            new 
java.util.ArrayList<>(vm.eventRequestManager().classPrepareRequests()));
+    vm.eventRequestManager()
+        .deleteEventRequests(
+            new 
java.util.ArrayList<>(vm.eventRequestManager().methodEntryRequests()));
+    final MethodExitRequest exit = 
vm.eventRequestManager().createMethodExitRequest();
+    exit.addClassFilter(className);
+    Assert.assertNotNull("An entry pause is required before selecting its 
exit", pausedThread);
+    exit.addThreadFilter(pausedThread);
+    exit.setSuspendPolicy(EventRequest.SUSPEND_EVENT_THREAD);
+    exit.enable();
+  }
+
+  /** Reads java.io.File's path without invoking code in the paused node. */
+  public static String filePath(Value value) {
+    final ObjectReference file = (ObjectReference) value;
+    return ((StringReference) 
file.getValue(file.referenceType().fieldByName("path"))).value();
+  }
+
+  @Override
+  public void close() {
+    try {
+      resume();
+      vm.dispose();
+    } catch (VMDisconnectedException expectedAfterCrash) {
+      // A forced node stop is the operation under test.
+    }
+  }
+}
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 64023379335..8520ee6a119 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -212,6 +212,7 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
@@ -284,6 +285,7 @@ public class DataRegion implements IDataRegionForQuery {
   private static final int MERGE_MOD_START_VERSION_NUM = 1;
 
   private static final Logger logger = 
LoggerFactory.getLogger(DataRegion.class);
+  private static final String LOAD_PENDING_SUFFIX = ".load-pending";
 
   /**
    * A read write lock for guaranteeing concurrent safety when accessing all 
fields in this class
@@ -955,6 +957,14 @@ public class DataRegion implements IDataRegionForQuery {
           } else {
             // some TsFileResource may be being persisted when the system 
crashed, try recovering
             // such resources
+            final File[] pendingLoads =
+                fsFactory.listFilesBySuffix(
+                    partitionFolder.getAbsolutePath(), TSFILE_SUFFIX + 
LOAD_PENDING_SUFFIX);
+            if (pendingLoads != null) {
+              for (File pendingLoad : pendingLoads) {
+                deleteUnpublishedLoad(pendingLoad);
+              }
+            }
             continueFailedRenames(partitionFolder, TEMP_SUFFIX);
             String partitionName = partitionFolder.getName();
             File[] tsFilesInThisFolder =
@@ -996,6 +1006,21 @@ public class DataRegion implements IDataRegionForQuery {
     }
   }
 
+  private void deleteUnpublishedLoad(File pendingFile) throws IOException {
+    final File targetFile =
+        new File(
+            pendingFile
+                .getPath()
+                .substring(0, pendingFile.getPath().length() - 
LOAD_PENDING_SUFFIX.length()));
+    if (!targetFile.exists()) {
+      Files.deleteIfExists(new File(targetFile + RESOURCE_SUFFIX).toPath());
+      
Files.deleteIfExists(ModificationFileV1.getNormalMods(targetFile).toPath());
+      
Files.deleteIfExists(ModificationFile.getExclusiveMods(targetFile).toPath());
+    }
+    // Keep the pending file until all sidecars are removed, so interrupted 
cleanup is retryable.
+    Files.deleteIfExists(pendingFile.toPath());
+  }
+
   /** check if the tsfile's time is smaller than system current time. */
   private void checkTsFileTime(File tsFile, long currentTime) throws 
DataRegionException {
     String[] items = tsFile.getName().replace(TSFILE_SUFFIX, 
"").split(FILE_NAME_SEPARATOR);
@@ -4845,75 +4870,82 @@ public class DataRegion implements IDataRegionForQuery {
     if (!targetFile.getParentFile().exists()) {
       targetFile.getParentFile().mkdirs();
     }
-    try {
-      if (deleteOriginFile) {
-        RetryUtils.retryOnException(
-            () -> {
-              FileUtils.moveFile(tsFileToLoad, targetFile);
-              return null;
-            });
-      } else {
-        // The staged bytes have to stay for a replica that has not applied 
this command yet, so
-        // they cannot be moved. A sealed TsFile is never written again, so a 
hard link gives this
-        // region its own path to the very same bytes instead of a second copy 
of them; the resource
-        // file below is updated by the load and is still copied.
-        RetryUtils.retryOnException(
-            () -> {
-              LoadUtil.transferFile(tsFileToLoad, targetFile, true);
-              return null;
-            });
-      }
-    } catch (final IOException e) {
-      logger.warn(
-          StorageEngineMessages
-              
.STORAGE_LOG_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_TARGET_28E43D85,
-          tsFileToLoad.getAbsolutePath(),
-          targetFile.getAbsolutePath(),
-          e);
-      throw new LoadFileException(
-          String.format(
-              StorageEngineMessages
-                  
.STORAGE_EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_S_TARGET_37BDA16F,
-              tsFileToLoad.getAbsolutePath(),
-              targetFile.getAbsolutePath(),
-              e.getMessage()));
-    }
-
+    // Recovery only discovers names ending in .tsfile. Keep the data 
invisible until its resource
+    // and every deletion are installed, including when hard-link creation 
falls back to a copy.
+    final File pendingFile = new File(targetFile + LOAD_PENDING_SUFFIX);
     final File resourceFileToLoad =
         fsFactory.getFile(tsFileToLoad.getAbsolutePath() + RESOURCE_SUFFIX);
     final File targetResourceFile =
         fsFactory.getFile(targetFile.getAbsolutePath() + RESOURCE_SUFFIX);
+    boolean loadPublished = false;
     try {
-      if (deleteOriginFile) {
+      try {
+        if (pendingFile.exists()) {
+          deleteUnpublishedLoad(pendingFile);
+        }
         RetryUtils.retryOnException(
             () -> {
-              FileUtils.moveFile(resourceFileToLoad, targetResourceFile);
+              LoadUtil.transferFile(tsFileToLoad, pendingFile, true);
               return null;
             });
-      } else {
+      } catch (final IOException e) {
+        logger.warn(
+            StorageEngineMessages
+                
.STORAGE_LOG_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_TARGET_28E43D85,
+            tsFileToLoad.getAbsolutePath(),
+            targetFile.getAbsolutePath(),
+            e);
+        throw new LoadFileException(
+            String.format(
+                StorageEngineMessages
+                    
.STORAGE_EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_TSFILE_ORIGIN_S_TARGET_37BDA16F,
+                tsFileToLoad.getAbsolutePath(),
+                targetFile.getAbsolutePath(),
+                e.getMessage()));
+      }
+
+      try {
         RetryUtils.retryOnException(
             () -> {
-              Files.copy(resourceFileToLoad.toPath(), 
targetResourceFile.toPath());
+              Files.copy(
+                  resourceFileToLoad.toPath(),
+                  targetResourceFile.toPath(),
+                  StandardCopyOption.REPLACE_EXISTING);
               return null;
             });
+      } catch (final IOException e) {
+        logger.warn(
+            StorageEngineMessages
+                
.STORAGE_LOG_FILE_RENAMING_FAILED_WHEN_LOADING_RESOURCE_FILE_ORIGIN_TARGET_9C22DDF3,
+            resourceFileToLoad.getAbsolutePath(),
+            targetResourceFile.getAbsolutePath(),
+            e);
+        throw new LoadFileException(
+            String.format(
+                StorageEngineMessages
+                    
.STORAGE_EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_RESOURCE_FILE_ORIGIN_S_9622AA6D,
+                resourceFileToLoad.getAbsolutePath(),
+                targetResourceFile.getAbsolutePath(),
+                e.getMessage()));
       }
-    } catch (final IOException e) {
-      logger.warn(
-          StorageEngineMessages
-              
.STORAGE_LOG_FILE_RENAMING_FAILED_WHEN_LOADING_RESOURCE_FILE_ORIGIN_TARGET_9C22DDF3,
-          resourceFileToLoad.getAbsolutePath(),
-          targetResourceFile.getAbsolutePath(),
-          e);
-      throw new LoadFileException(
-          String.format(
-              StorageEngineMessages
-                  
.STORAGE_EXCEPTION_FILE_RENAMING_FAILED_WHEN_LOADING_RESOURCE_FILE_ORIGIN_S_9622AA6D,
-              resourceFileToLoad.getAbsolutePath(),
-              targetResourceFile.getAbsolutePath(),
-              e.getMessage()));
-    }
 
-    loadModFile(tsFileToLoad, targetFile, deleteOriginFile, tsFileResource);
+      // Keep all source files intact until the complete destination is 
published.
+      loadModFile(tsFileToLoad, targetFile, false, tsFileResource);
+      try {
+        Files.move(pendingFile.toPath(), targetFile.toPath(), 
StandardCopyOption.ATOMIC_MOVE);
+      } catch (IOException e) {
+        throw new LoadFileException(e);
+      }
+      loadPublished = true;
+    } finally {
+      if (!loadPublished && pendingFile.exists()) {
+        try {
+          deleteUnpublishedLoad(pendingFile);
+        } catch (IOException e) {
+          logger.warn(StorageEngineMessages.FAILED_TO_DELETE_FILE_OR_DIR, 
pendingFile, e);
+        }
+      }
+    }
 
     // Listen before the tsFile is added into tsFile manager to avoid it being 
compacted
     PipeInsertionDataNodeListener.getInstance()
@@ -4921,6 +4953,26 @@ public class DataRegion implements IDataRegionForQuery {
 
     tsFileManager.add(tsFileResource, false);
 
+    if (deleteOriginFile) {
+      for (File sourceFile :
+          new File[] {
+            tsFileToLoad,
+            resourceFileToLoad,
+            ModificationFileV1.getNormalMods(tsFileToLoad),
+            ModificationFile.getExclusiveMods(tsFileToLoad)
+          }) {
+        try {
+          RetryUtils.retryOnException(
+              () -> {
+                Files.deleteIfExists(sourceFile.toPath());
+                return null;
+              });
+        } catch (IOException e) {
+          throw new LoadFileException(e);
+        }
+      }
+    }
+
     return true;
   }
 
@@ -4933,7 +4985,9 @@ public class DataRegion implements IDataRegionForQuery {
       final File oldTargetModFile = 
ModificationFileV1.getNormalMods(targetTsFile);
       moveModFile(oldModFileToLoad, oldTargetModFile, deleteOriginFile);
       try {
-        tsFileResource.upgradeModFile(upgradeModFileThreadPool);
+        // Publishing must wait for a successful conversion; a background 
failure must not be
+        // swallowed by getExclusiveModFile and expose an incomplete deletion 
set after restart.
+        tsFileResource.upgradeModFile(null);
       } catch (IOException e) {
         throw new LoadFileException(e);
       }
@@ -4941,6 +4995,9 @@ public class DataRegion implements IDataRegionForQuery {
       final File newTargetModFile = 
ModificationFile.getExclusiveMods(targetTsFile);
       moveModFile(newModFileToLoad, newTargetModFile, deleteOriginFile);
     }
+    // The resource may have cached a source-side handle before its TsFile 
path was changed.
+    // Subsequent reads must use the installed deletions, including after 
source cleanup.
+    
tsFileResource.setExclusiveModFile(ModificationFile.getExclusiveMods(tsFileResource));
     // force update mod file metrics
     tsFileResource.getExclusiveModFile();
   }
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
index c82a65d7389..90b133258a5 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
@@ -42,6 +42,7 @@ import 
org.apache.iotdb.db.exception.DataTypeInconsistentException;
 import org.apache.iotdb.db.exception.TsFileProcessorException;
 import org.apache.iotdb.db.exception.WriteProcessException;
 import org.apache.iotdb.db.exception.WriteProcessRejectException;
+import org.apache.iotdb.db.exception.load.LoadFileException;
 import org.apache.iotdb.db.queryengine.common.QueryId;
 import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
 import 
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
@@ -77,6 +78,7 @@ import 
org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
 import 
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
 import 
org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry;
 import 
org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
 import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
@@ -105,7 +107,9 @@ import org.apache.tsfile.read.expression.QueryExpression;
 import org.apache.tsfile.read.query.dataset.QueryDataSet;
 import org.apache.tsfile.read.reader.IPointReader;
 import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.TsFileWriter;
 import org.apache.tsfile.write.record.TSRecord;
+import org.apache.tsfile.write.record.Tablet;
 import org.apache.tsfile.write.record.datapoint.DataPoint;
 import org.apache.tsfile.write.schema.MeasurementSchema;
 import org.junit.After;
@@ -120,11 +124,13 @@ import org.slf4j.LoggerFactory;
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Field;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
@@ -235,6 +241,85 @@ public class DataRegionTest {
     return insertRowNode;
   }
 
+  @Test
+  public void testLoadInstallationRetainsOrDeletesTheCompleteSourceGroup() 
throws Exception {
+    for (boolean deleteOrigin : new boolean[] {false, true}) {
+      for (boolean withMods : new boolean[] {false, true}) {
+        final File sourceDir =
+            new File(TestConstant.OUTPUT_DATA_DIR, "load-source-" + 
deleteOrigin + "-" + withMods);
+        Files.createDirectories(sourceDir.toPath());
+        final File source = new File(sourceDir, "1-1-0-0.tsfile");
+        final MeasurementSchema schema = new MeasurementSchema(measurementId, 
TSDataType.INT32);
+        try (TsFileWriter writer = new TsFileWriter(source)) {
+          writer.registerAlignedTimeseries(deviceId, 
Collections.singletonList(schema));
+          final Tablet tablet = new Tablet(deviceId, 
Collections.singletonList(schema), 3);
+          for (int row = 0; row < 3; row++) {
+            tablet.addTimestamp(row, row + 1L);
+            tablet.addValue(row, measurementId, row);
+          }
+          writer.writeAligned(tablet);
+        }
+        final TsFileResource resource = new TsFileResource(source);
+        resource.setStatus(TsFileResourceStatus.NORMAL);
+        resource.updateStartTime(device, 1);
+        resource.updateEndTime(device, 3);
+        resource.serialize();
+        final File sourceMods = ModificationFile.getExclusiveMods(source);
+        if (withMods) {
+          try (ModificationFile mods = resource.getExclusiveModFile()) {
+            mods.write(
+                new TreeDeletionEntry(new MeasurementPath(deviceId + "." + 
measurementId), 2, 2));
+          }
+        }
+        final byte[] data = Files.readAllBytes(source.toPath());
+        final byte[] deletions = withMods ? 
Files.readAllBytes(sourceMods.toPath()) : null;
+        final File sourceResource = new File(source + 
TsFileResource.RESOURCE_SUFFIX);
+        final File savedResource = new File(sourceDir, "saved-resource");
+        Files.move(sourceResource.toPath(), savedResource.toPath());
+        try {
+          final LoadFileException failure =
+              Assert.assertThrows(
+                  LoadFileException.class,
+                  () ->
+                      dataRegion.loadNewTsFile(
+                          resource, deleteOrigin, false, true, 
Optional.empty()));
+          Assert.assertTrue(
+              failure.getMessage(),
+              failure.getMessage().contains(sourceResource.getAbsolutePath()));
+          final File rejectedTarget = resource.getTsFile();
+          Assert.assertNotEquals(source.getAbsolutePath(), 
rejectedTarget.getAbsolutePath());
+          Assert.assertFalse(rejectedTarget.exists());
+          Assert.assertFalse(new File(rejectedTarget + 
".load-pending").exists());
+          Assert.assertFalse(new File(rejectedTarget + 
TsFileResource.RESOURCE_SUFFIX).exists());
+          
Assert.assertFalse(ModificationFile.getExclusiveMods(rejectedTarget).exists());
+          Assert.assertArrayEquals(data, Files.readAllBytes(source.toPath()));
+          Assert.assertEquals(withMods, sourceMods.exists());
+        } finally {
+          Files.move(savedResource.toPath(), sourceResource.toPath());
+          resource.setFile(source);
+        }
+        dataRegion.loadNewTsFile(resource, deleteOrigin, false, true, 
Optional.empty());
+        final File installed = resource.getTsFile();
+        Assert.assertArrayEquals(data, Files.readAllBytes(installed.toPath()));
+        Assert.assertTrue(new File(installed + 
TsFileResource.RESOURCE_SUFFIX).isFile());
+        Assert.assertFalse(new File(installed + ".load-pending").exists());
+        final File installedMods = 
ModificationFile.getExclusiveMods(installed);
+        Assert.assertEquals(
+            installedMods.getCanonicalFile(),
+            resource.getExclusiveModFile().getFile().getCanonicalFile());
+        Assert.assertEquals(withMods ? 1 : 0, 
resource.getExclusiveModFile().getAllMods().size());
+        Assert.assertEquals(withMods, installedMods.exists());
+        if (withMods) {
+          Assert.assertArrayEquals(deletions, 
Files.readAllBytes(installedMods.toPath()));
+        }
+        Assert.assertEquals(!deleteOrigin, source.exists());
+        Assert.assertEquals(
+            !deleteOrigin, new File(source + 
TsFileResource.RESOURCE_SUFFIX).exists());
+        Assert.assertEquals(withMods && !deleteOrigin, sourceMods.exists());
+      }
+    }
+  }
+
   @Test
   public void testRejectedRowFlushesOnlySelectedOtherMemtable() throws 
Exception {
     assertRejectedInsertionFlushesOnlySelectedOtherMemtable(false);

Reply via email to