This is an automated email from the ASF dual-hosted git repository.

Caideyipi pushed a commit to branch fix/load-mini-tsfile-20261118
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 0fd37369e68f07f49292f95d91ae26c72ad34a3c
Author: Caideyipi <[email protected]>
AuthorDate: Wed Nov 18 16:10:15 2026 +0800

    fix(load): validate tree mini-file permissions before conversion
---
 .../it/env/cluster/config/MppDataNodeConfig.java   |   7 +
 .../it/env/remote/config/RemoteDataNodeConfig.java |   5 +
 .../apache/iotdb/itbase/env/DataNodeConfig.java    |   2 +
 .../iotdb/db/it/IoTDBLoadTsFileBoundaryIT.java     | 769 +++++++++++++++++++++
 .../plan/analyze/load/LoadTsFileAnalyzer.java      | 112 ++-
 .../converter/LoadTsFileDataTypeConverter.java     |   2 +-
 6 files changed, 861 insertions(+), 36 deletions(-)

diff --git 
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java
 
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java
index 95efb88b754..a15feddb4e0 100644
--- 
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java
+++ 
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java
@@ -56,6 +56,13 @@ public class MppDataNodeConfig extends MppBaseConfig 
implements DataNodeConfig {
     return this;
   }
 
+  @Override
+  public DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean 
asyncUpdate) {
+    properties.setProperty(
+        "dn_metric_prometheus_reporter_async_update", 
String.valueOf(asyncUpdate));
+    return this;
+  }
+
   @Override
   public DataNodeConfig setMetricPrometheusReporterUsername(String username) {
     properties.setProperty("metric_prometheus_reporter_username", username);
diff --git 
a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java
 
b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java
index d8350577f57..a1db5cfc04e 100644
--- 
a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java
+++ 
b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java
@@ -28,6 +28,11 @@ public class RemoteDataNodeConfig implements DataNodeConfig {
     return this;
   }
 
+  @Override
+  public DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean 
asyncUpdate) {
+    return this;
+  }
+
   @Override
   public DataNodeConfig setMetricPrometheusReporterUsername(String username) {
     return this;
diff --git 
a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java
 
b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java
index 99c360d2678..183eadcac8b 100644
--- 
a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java
+++ 
b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java
@@ -25,6 +25,8 @@ import java.util.List;
 public interface DataNodeConfig {
   DataNodeConfig setMetricReporterType(List<String> metricReporterTypes);
 
+  DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean asyncUpdate);
+
   DataNodeConfig setMetricPrometheusReporterUsername(String username);
 
   DataNodeConfig setMetricPrometheusReporterPassword(String password);
diff --git 
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileBoundaryIT.java
 
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileBoundaryIT.java
new file mode 100644
index 00000000000..10710ef6400
--- /dev/null
+++ 
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileBoundaryIT.java
@@ -0,0 +1,769 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.commons.auth.entity.PrivilegeType;
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.iotdb.commons.path.PartialPath;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.StringArrayDeviceID;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.schema.Schema;
+import org.apache.tsfile.write.writer.TsFileIOWriter;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.createUser;
+import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSeriesPrivilege;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({LocalStandaloneIT.class, ClusterIT.class})
+public class IoTDBLoadTsFileBoundaryIT {
+  private static final String MINI_DATABASE = "root.load_p115";
+  private static final String MIXED_DATABASE = "root.load_p115_mixed";
+  private static final String ALL_MINI_DATABASE = "root.load_p115_all_mini";
+  private static final String INVALID_DATABASE = "root.load_p120";
+
+  private File tmpDir;
+
+  @BeforeClass
+  public static void setUpClass() throws Exception {
+    
EnvFactory.getEnv().getConfig().getCommonConfig().setTimePartitionInterval(10_000L);
+    
EnvFactory.getEnv().getConfig().getCommonConfig().setEnforceStrongPassword(false);
+    EnvFactory.getEnv()
+        .getConfig()
+        .getConfigNodeConfig()
+        .setMetricReporterType(Collections.singletonList("PROMETHEUS"));
+    EnvFactory.getEnv()
+        .getConfig()
+        .getDataNodeConfig()
+        .setMetricReporterType(Collections.singletonList("PROMETHEUS"))
+        .setMetricPrometheusReporterAsyncUpdate(false);
+    EnvFactory.getEnv().initClusterEnvironment();
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    tmpDir = Files.createTempDirectory("load-boundary").toFile();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+      deleteDatabase(statement, MINI_DATABASE);
+      for (int i = 0; i < 4; i++) {
+        deleteDatabase(statement, MINI_DATABASE + "_threshold_" + i);
+      }
+      deleteDatabase(statement, MIXED_DATABASE);
+      deleteDatabase(statement, ALL_MINI_DATABASE);
+      deleteDatabase(statement, INVALID_DATABASE);
+    } catch (SQLException ignored) {
+      // The cluster may already be unavailable after a failed test.
+    } finally {
+      deleteRecursively(tmpDir);
+    }
+  }
+
+  @AfterClass
+  public static void tearDownClass() {
+    EnvFactory.getEnv().cleanClusterEnvironment();
+  }
+
+  @Test
+  public void testMiniFileThresholdAndMixedStatementReconstruction() throws 
Exception {
+    for (int thresholdIndex = 0; thresholdIndex < 4; thresholdIndex++) {
+      final String thresholdDatabase = MINI_DATABASE + "_threshold_" + 
thresholdIndex;
+      final String thresholdDevice = thresholdDatabase + ".threshold_device";
+      final File thresholdDir = new File(tmpDir, "threshold-" + 
thresholdIndex);
+      Assert.assertTrue(thresholdDir.mkdir());
+      final File thresholdFile = new File(thresholdDir, "threshold.tsfile");
+      writeTreeFile(
+          thresholdFile, thresholdDevice, new long[] {9999, 10000, 20000}, new 
long[] {9, 10, 20});
+      writeTreeDeletion(thresholdFile, thresholdDevice, "value", 9999, 9999);
+      final long fileSize = thresholdFile.length();
+      final long threshold = thresholdIndex == 0 ? -1 : fileSize + 
thresholdIndex - 2;
+
+      final long conversionsBefore = getMiniConversionCount();
+      try (Connection connection = EnvFactory.getEnv().getConnection();
+          Statement statement = connection.createStatement()) {
+        statement.execute("create database " + thresholdDatabase);
+        statement.execute("create timeseries " + thresholdDevice + ".value 
INT64");
+        statement.execute(
+            String.format(
+                "load \"%s\" with 
('database-level'='2','auto-create-schema'='false',"
+                    + 
"'on-success'='none','tablet-conversion-threshold'='%d')",
+                thresholdFile.getAbsolutePath(), threshold));
+        assertThresholdFileContents(statement, thresholdDevice);
+      }
+
+      final long expectedConversionDelta = thresholdIndex >= 2 ? 1 : 0;
+      Assert.assertEquals(
+          "The load_time_cost analysis_cast_tablets metric must prove the 
selected path",
+          expectedConversionDelta,
+          getMiniConversionCount() - conversionsBefore);
+      Assert.assertTrue(thresholdFile.exists());
+      Assert.assertTrue(
+          new File(thresholdFile.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX).exists());
+
+      try (Connection connection = EnvFactory.getEnv().getConnection();
+          Statement statement = connection.createStatement()) {
+        deleteDatabase(statement, thresholdDatabase);
+      }
+    }
+
+    final File deleteOnSuccessFile = new File(tmpDir, 
"threshold-delete-on-success.tsfile");
+    final String deleteOnSuccessDevice = MINI_DATABASE + ".delete_on_success";
+    writeTreeFile(
+        deleteOnSuccessFile,
+        deleteOnSuccessDevice,
+        new long[] {9999, 10000, 20000},
+        new long[] {9, 10, 20});
+    writeTreeDeletion(deleteOnSuccessFile, deleteOnSuccessDevice, "value", 
9999, 9999);
+    final long deleteConversionBefore = getMiniConversionCount();
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+      statement.execute("create database " + MINI_DATABASE);
+      statement.execute("create timeseries " + deleteOnSuccessDevice + ".value 
INT64");
+      statement.execute(
+          String.format(
+              "load \"%s\" with 
('database-level'='2','auto-create-schema'='false',"
+                  + 
"'on-success'='delete','tablet-conversion-threshold'='%d')",
+              deleteOnSuccessFile.getAbsolutePath(), 
deleteOnSuccessFile.length()));
+      assertThresholdFileContents(statement, deleteOnSuccessDevice);
+    }
+    Assert.assertEquals(1, getMiniConversionCount() - deleteConversionBefore);
+    Assert.assertFalse(deleteOnSuccessFile.exists());
+    Assert.assertFalse(
+        new File(deleteOnSuccessFile.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX).exists());
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+      deleteDatabase(statement, MINI_DATABASE);
+    }
+
+    final File mixedDir = new File(tmpDir, "mixed");
+    Assert.assertTrue(mixedDir.mkdir());
+    final File firstMini = new File(mixedDir, "01-first-mini.tsfile");
+    final File firstNormal = new File(mixedDir, "02-first-normal.tsfile");
+    final File middleMini = new File(mixedDir, "03-middle-mini.tsfile");
+    final File lastNormal = new File(mixedDir, "04-last-normal.tsfile");
+    final File lastMini = new File(mixedDir, "05-last-mini.tsfile");
+    writeTreeFile(firstMini, MIXED_DATABASE + ".small", new long[] {1}, new 
long[] {11});
+    writeTreeFile(
+        firstNormal, MIXED_DATABASE + ".normal_first", buildTimes(64, 100), 
buildValues(64, 1000));
+    writeTreeFile(middleMini, MIXED_DATABASE + ".middle", new long[] {2}, new 
long[] {22});
+    writeTreeFile(
+        lastNormal, MIXED_DATABASE + ".normal_last", buildTimes(64, 100), 
buildValues(64, 2000));
+    writeTreeFile(lastMini, MIXED_DATABASE + ".tail", new long[] {2}, new 
long[] {33});
+
+    assertTsFileOrder(
+        mixedDir,
+        "01-first-mini.tsfile",
+        "02-first-normal.tsfile",
+        "03-middle-mini.tsfile",
+        "04-last-normal.tsfile",
+        "05-last-mini.tsfile");
+    final long threshold =
+        Math.max(Math.max(firstMini.length(), middleMini.length()), 
lastMini.length());
+    Assert.assertTrue(
+        "The normal files must remain above the conversion threshold",
+        firstNormal.length() > threshold && lastNormal.length() > threshold);
+    final long conversionsBefore = getMiniConversionCount();
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+      statement.execute("create database " + MIXED_DATABASE);
+      for (final String device :
+          List.of(
+              MIXED_DATABASE + ".small",
+              MIXED_DATABASE + ".normal_first",
+              MIXED_DATABASE + ".middle",
+              MIXED_DATABASE + ".normal_last",
+              MIXED_DATABASE + ".tail")) {
+        statement.execute("create timeseries " + device + ".value INT64");
+      }
+      statement.execute(buildLoadSql(mixedDir, threshold, "2", "none"));
+      assertOneTreeValue(statement, MIXED_DATABASE + ".small", 1, 11);
+      assertOneTreeValue(statement, MIXED_DATABASE + ".middle", 2, 22);
+      assertOneTreeValue(statement, MIXED_DATABASE + ".tail", 2, 33);
+      assertTreeSeriesValues(statement, MIXED_DATABASE + ".normal_first", 64, 
1000);
+      assertTreeSeriesValues(statement, MIXED_DATABASE + ".normal_last", 64, 
2000);
+    }
+    Assert.assertEquals(3, getMiniConversionCount() - conversionsBefore);
+    Assert.assertTrue(firstMini.exists());
+    Assert.assertTrue(firstNormal.exists());
+    Assert.assertTrue(middleMini.exists());
+    Assert.assertTrue(lastNormal.exists());
+    Assert.assertTrue(lastMini.exists());
+
+    final File allMiniDir = new File(tmpDir, "all-mini");
+    Assert.assertTrue(allMiniDir.mkdir());
+    final File[] allMiniFiles = {
+      new File(allMiniDir, "01-one.tsfile"),
+      new File(allMiniDir, "02-two.tsfile"),
+      new File(allMiniDir, "03-three.tsfile")
+    };
+    for (int i = 0; i < allMiniFiles.length; i++) {
+      writeTreeFile(
+          allMiniFiles[i],
+          ALL_MINI_DATABASE + ".device_" + i,
+          new long[] {i + 1},
+          new long[] {100 + i});
+    }
+    assertTsFileOrder(allMiniDir, "01-one.tsfile", "02-two.tsfile", 
"03-three.tsfile");
+    final long allMiniThreshold =
+        
Arrays.stream(allMiniFiles).mapToLong(File::length).max().orElseThrow();
+    final long allMiniConversionsBefore = getMiniConversionCount();
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+      statement.execute("create database " + ALL_MINI_DATABASE);
+      for (int i = 0; i < allMiniFiles.length; i++) {
+        statement.execute(
+            "create timeseries " + ALL_MINI_DATABASE + ".device_" + i + 
".value INT64");
+      }
+      statement.execute(buildLoadSql(allMiniDir, allMiniThreshold, "2", 
"none"));
+      for (int i = 0; i < allMiniFiles.length; i++) {
+        assertOneTreeValue(statement, ALL_MINI_DATABASE + ".device_" + i, i + 
1, 100 + i);
+        Assert.assertTrue(allMiniFiles[i].exists());
+      }
+    }
+    Assert.assertEquals(3, getMiniConversionCount() - 
allMiniConversionsBefore);
+  }
+
+  @Test
+  public void testMiniFileSuccessDoesNotHideFollowingFailure() throws 
Exception {
+    final File loadDir = new File(tmpDir, "mini-before-failure");
+    Assert.assertTrue(loadDir.mkdir());
+    final String smallDevice = MIXED_DATABASE + ".small_before_failure";
+    final String failingDevice = MIXED_DATABASE + ".schema_mismatch";
+    final String unprocessedDevice = MIXED_DATABASE + ".mini_after_failure";
+    final File firstMini = new File(loadDir, "01-mini.tsfile");
+    final File failingNormal = new File(loadDir, "02-normal-failure.tsfile");
+    final File tailMini = new File(loadDir, "03-tail-mini.tsfile");
+    writeTreeFile(firstMini, smallDevice, new long[] {1}, new long[] {11});
+    final long[] failingTimes = buildTimes(256, 10);
+    final Object[] failingValues = new Object[failingTimes.length];
+    final Random random = new Random(0);
+    for (int i = 0; i < failingValues.length; i++) {
+      failingValues[i] = random.nextDouble();
+    }
+    writeTypedTreeFile(
+        failingNormal, failingDevice, TSDataType.DOUBLE, failingTimes, 
failingValues);
+    writeTreeFile(tailMini, unprocessedDevice, new long[] {2}, new long[] 
{22});
+    assertTsFileOrder(loadDir, "01-mini.tsfile", "02-normal-failure.tsfile", 
"03-tail-mini.tsfile");
+
+    final long threshold = Math.max(firstMini.length(), tailMini.length());
+    Assert.assertTrue(
+        String.format(
+            "Normal failure file (%d bytes) must exceed mini threshold (%d 
bytes)",
+            failingNormal.length(), threshold),
+        failingNormal.length() > threshold);
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+      statement.execute("create database " + MIXED_DATABASE);
+      statement.execute("create timeseries " + smallDevice + ".value INT64");
+      statement.execute("create timeseries " + failingDevice + ".value INT32");
+      statement.execute("create timeseries " + unprocessedDevice + ".value 
INT64");
+    }
+
+    final long conversionsBefore = getMiniConversionCount();
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+      try {
+        statement.execute(
+            String.format(
+                "load \"%s\" with 
('database-level'='1','tablet-conversion-threshold'='%d',"
+                    + 
"'auto-create-schema'='false','convert-on-type-mismatch'='false',"
+                    + "'on-success'='none')",
+                loadDir.getAbsolutePath(), threshold));
+        Assert.fail("The later type mismatch must fail the whole LOAD 
statement");
+      } catch (SQLException expected) {
+        Assert.assertTrue(
+            "The type mismatch for the middle file must be the reported 
failure: "
+                + expected.getMessage(),
+            expected.getMessage().contains(failingDevice));
+      }
+      assertOneTreeValue(statement, smallDevice, 1, 11);
+      try (ResultSet resultSet =
+          statement.executeQuery("select count(value) from " + failingDevice)) 
{
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(0, resultSet.getLong(1));
+      }
+      try (ResultSet resultSet =
+          statement.executeQuery("select count(value) from " + 
unprocessedDevice)) {
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(0, resultSet.getLong(1));
+      }
+    }
+    Assert.assertEquals(1, getMiniConversionCount() - conversionsBefore);
+    Assert.assertTrue("on-success=none must retain the successful mini file", 
firstMini.exists());
+    Assert.assertTrue("The failed source file must be retained", 
failingNormal.exists());
+    Assert.assertTrue("Files after the failed entry must remain unprocessed", 
tailMini.exists());
+  }
+
+  @Test
+  public void testMiniFileChecksSubmittingUser() throws Exception {
+    final String device = MINI_DATABASE + ".permission_probe";
+    final String user = "load_p115_user";
+    final String password = "test123123456";
+    final File tsFile = new File(tmpDir, "mini-user.tsfile");
+    writeTreeFile(tsFile, device, new long[] {1}, new long[] {42});
+    writeTreeDeletion(tsFile, device, "value", 2, 2);
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+      statement.execute("create database " + MINI_DATABASE);
+      statement.execute("create timeseries " + device + ".value INT64");
+      createUser(user, password);
+      try {
+        final String loadSql =
+            String.format(
+                "load \"%s\" with 
('auto-create-schema'='false','verify'='false',"
+                    + 
"'on-success'='delete','tablet-conversion-threshold'='%d')",
+                tsFile.getAbsolutePath(), tsFile.length());
+        final long conversionsBefore = getMiniConversionCount();
+        try (Connection userConnection = 
EnvFactory.getEnv().getConnection(user, password);
+            Statement userStatement = userConnection.createStatement()) {
+          try {
+            userStatement.execute(loadSql);
+            Assert.fail("Mini-file conversion must check the submitting user's 
WRITE_DATA");
+          } catch (SQLException expected) {
+            Assert.assertEquals(
+                TSStatusCode.NO_PERMISSION.getStatusCode(), 
expected.getErrorCode());
+          }
+        }
+        Assert.assertEquals(1, getMiniConversionCount() - conversionsBefore);
+        Assert.assertTrue(tsFile.exists());
+        Assert.assertTrue(
+            new File(tsFile.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX).exists());
+        try (ResultSet resultSet = statement.executeQuery("select count(value) 
from " + device)) {
+          Assert.assertTrue(resultSet.next());
+          Assert.assertEquals(0, resultSet.getLong(1));
+        }
+        grantUserSeriesPrivilege(user, PrivilegeType.WRITE_DATA, MINI_DATABASE 
+ ".**");
+        try (Connection userConnection = 
EnvFactory.getEnv().getConnection(user, password);
+            Statement userStatement = userConnection.createStatement()) {
+          userStatement.execute(loadSql);
+        }
+        Assert.assertEquals(2, getMiniConversionCount() - conversionsBefore);
+        assertOneTreeValue(statement, device, 1, 42);
+        Assert.assertFalse(tsFile.exists());
+        Assert.assertFalse(
+            new File(tsFile.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX).exists());
+      } finally {
+        statement.execute("drop user " + user);
+      }
+    }
+  }
+
+  @Test
+  public void testInvalidTreeDevicePathFailsWithAndWithoutMiniConversion() 
throws Exception {
+    final long loadMemoryBefore = getLoadTsFileUsedMemory();
+    for (final boolean nullNode : new boolean[] {false, true}) {
+      final StringArrayDeviceID invalidDevice =
+          new StringArrayDeviceID(
+              nullNode
+                  ? new String[] {"root.load_p120", null, "invalid_device"}
+                  : new String[] {"root", "load_p120", ""});
+      final File invalidFile = new File(tmpDir, "invalid-path-" + nullNode + 
".tsfile");
+      writeInvalidPathTsFile(invalidFile, invalidDevice);
+      writeTreeDeletion(invalidFile, INVALID_DATABASE + ".valid_device", 
"value", 1, 1);
+
+      for (final boolean miniConversion : new boolean[] {false, true}) {
+        final File inputDir =
+            new File(tmpDir, (miniConversion ? "invalid-mini-" : 
"invalid-normal-") + nullNode);
+        Assert.assertTrue(inputDir.mkdir());
+        final File inputFile = new File(inputDir, "invalid-path.tsfile");
+        Files.copy(invalidFile.toPath(), inputFile.toPath());
+        Files.copy(
+            new File(invalidFile.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX).toPath(),
+            new File(inputFile.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX).toPath());
+
+        final long conversionsBefore = getMiniConversionCount();
+        try (Connection connection = EnvFactory.getEnv().getConnection();
+            Statement statement = connection.createStatement()) {
+          deleteDatabase(statement, INVALID_DATABASE);
+          statement.execute("create database " + INVALID_DATABASE);
+          statement.execute(
+              "create aligned timeseries "
+                  + INVALID_DATABASE
+                  + ".valid_device(quality INT32, value DOUBLE)");
+          final String attributes =
+              miniConversion
+                  ? 
"'verify'='false','auto-create-schema'='false','on-success'='delete',"
+                      + "'tablet-conversion-threshold'='"
+                      + inputFile.length()
+                      + "'"
+                  : 
"'verify'='false','auto-create-schema'='false','on-success'='delete'";
+          try {
+            statement.execute(
+                String.format("load \"%s\" with (%s)", 
inputFile.getAbsolutePath(), attributes));
+            Assert.fail("A TsFile with an empty Tree device path must fail 
LOAD");
+          } catch (SQLException expected) {
+            Assert.assertTrue(
+                "The error must identify the illegal empty node in the TsFile 
device path",
+                expected
+                    .getMessage()
+                    .contains(
+                        new IllegalPathException(new 
PartialPath(invalidDevice).getFullPath())
+                            .getMessage()));
+            Assert.assertEquals(
+                TSStatusCode.LOAD_FILE_ERROR.getStatusCode(), 
expected.getErrorCode());
+          }
+          Assert.assertEquals(
+              "Only the mini-file case should enter tablet-conversion 
analysis",
+              miniConversion ? 1 : 0,
+              getMiniConversionCount() - conversionsBefore);
+
+          Assert.assertTrue("Failed LOAD must retain its source TsFile", 
inputFile.exists());
+          Assert.assertTrue(
+              "Failed LOAD must retain the source mods",
+              new File(inputFile.getAbsolutePath() + 
ModificationFile.FILE_SUFFIX).exists());
+          try (ResultSet resultSet =
+              statement.executeQuery("show timeseries " + INVALID_DATABASE + 
".**")) {
+            final Set<String> actualTimeseries = new HashSet<>();
+            while (resultSet.next()) {
+              Assert.assertTrue(
+                  "SHOW TIMESERIES returned a duplicate path",
+                  actualTimeseries.add(resultSet.getString(1)));
+            }
+            Assert.assertEquals(
+                Set.of(
+                    INVALID_DATABASE + ".valid_device.quality",
+                    INVALID_DATABASE + ".valid_device.value"),
+                actualTimeseries);
+          }
+          try (ResultSet resultSet =
+              statement.executeQuery(
+                  "select quality, value from " + INVALID_DATABASE + 
".valid_device")) {
+            if (miniConversion) {
+              Assert.assertTrue(
+                  "Mini conversion must retain the successfully converted 
valid prefix",
+                  resultSet.next());
+              Assert.assertEquals(1, resultSet.getLong(1));
+              Assert.assertEquals(7, resultSet.getInt(2));
+              Assert.assertNull(
+                  "Source mods must also apply to a visible valid prefix", 
resultSet.getObject(3));
+              Assert.assertFalse(resultSet.next());
+            } else {
+              Assert.assertFalse(
+                  "Normal analysis failure must not execute the LOAD data 
phase", resultSet.next());
+            }
+          }
+        }
+      }
+    }
+
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+      deleteDatabase(statement, INVALID_DATABASE);
+      statement.execute("create database " + INVALID_DATABASE);
+      statement.execute("create timeseries " + INVALID_DATABASE + 
".valid_after.value INT64");
+      final File validFile = new File(tmpDir, "valid-after-invalid.tsfile");
+      writeTreeFile(validFile, INVALID_DATABASE + ".valid_after", new long[] 
{1}, new long[] {42});
+      statement.execute(
+          String.format(
+              "load \"%s\" with 
('verify'='false','auto-create-schema'='false','on-success'='none')",
+              validFile.getAbsolutePath()));
+      assertOneTreeValue(statement, INVALID_DATABASE + ".valid_after", 1, 42);
+    }
+    assertLoadTsFileMemoryReturnsTo(loadMemoryBefore);
+  }
+
+  private String buildLoadSql(
+      final File file,
+      final long tabletConversionThreshold,
+      final String databaseLevel,
+      final String onSuccess) {
+    final String thresholdAttribute =
+        tabletConversionThreshold < 0
+            ? ""
+            : ", 'tablet-conversion-threshold'='" + tabletConversionThreshold 
+ "'";
+    return String.format(
+        "load \"%s\" with ('database-level'='%s','auto-create-schema'='false',"
+            + "'on-success'='%s'%s)",
+        file.getAbsolutePath(), databaseLevel, onSuccess, thresholdAttribute);
+  }
+
+  private long getMiniConversionCount() {
+    long count = 0;
+    for (final String content : 
EnvFactory.getEnv().getMetricPrometheusReporterContents(null)) {
+      if (content == null) {
+        continue;
+      }
+      for (final String line : content.split("\\n")) {
+        final int labelStart = line.indexOf('{');
+        if (labelStart < 0
+            || !line.substring(0, labelStart).endsWith("_count")
+            || !line.contains("load_time_cost")
+            || !line.contains("name=\"analysis_cast_tablets\"")) {
+          continue;
+        }
+        count += 
Math.round(Double.parseDouble(line.substring(line.lastIndexOf(' ') + 
1).trim()));
+      }
+    }
+    return count;
+  }
+
+  private long getLoadTsFileUsedMemory() {
+    long usedMemory = 0;
+    boolean foundMetric = false;
+    for (final String content : 
EnvFactory.getEnv().getMetricPrometheusReporterContents(null)) {
+      if (content == null) {
+        continue;
+      }
+      for (final String line : content.split("\\n")) {
+        final int labelStart = line.indexOf('{');
+        if (labelStart < 0
+            || !line.substring(0, labelStart).equals("load_mem")
+            || !line.contains("name=\"LoadTsFileUsedMemory\"")) {
+          continue;
+        }
+        foundMetric = true;
+        usedMemory += 
Math.round(Double.parseDouble(line.substring(line.lastIndexOf(' ') + 1)));
+      }
+    }
+    Assert.assertTrue(
+        "LoadTsFileUsedMemory must be exposed by the configured reporter", 
foundMetric);
+    return usedMemory;
+  }
+
+  private void assertLoadTsFileMemoryReturnsTo(final long expectedMemory)
+      throws InterruptedException {
+    final long deadline = System.currentTimeMillis() + 10_000L;
+    long actualMemory = getLoadTsFileUsedMemory();
+    while (actualMemory != expectedMemory && System.currentTimeMillis() < 
deadline) {
+      Thread.sleep(100L);
+      actualMemory = getLoadTsFileUsedMemory();
+    }
+    Assert.assertEquals(
+        "LOAD memory usage should return to its prior level", expectedMemory, 
actualMemory);
+  }
+
+  private void assertThresholdFileContents(final Statement statement, final 
String device)
+      throws SQLException {
+    try (ResultSet resultSet =
+        statement.executeQuery("select value from " + device + " order by 
time")) {
+      Assert.assertTrue(resultSet.next());
+      Assert.assertEquals(10000, resultSet.getLong(1));
+      Assert.assertEquals(10, resultSet.getLong(2));
+      Assert.assertTrue(resultSet.next());
+      Assert.assertEquals(20000, resultSet.getLong(1));
+      Assert.assertEquals(20, resultSet.getLong(2));
+      Assert.assertFalse(resultSet.next());
+    }
+  }
+
+  private void assertTreeSeriesValues(
+      final Statement statement,
+      final String device,
+      final int expectedCount,
+      final long startValue)
+      throws SQLException {
+    try (ResultSet resultSet =
+        statement.executeQuery("select value from " + device + " order by 
time")) {
+      for (int i = 0; i < expectedCount; i++) {
+        Assert.assertTrue(
+            "A normal-path row was lost during statement reconstruction", 
resultSet.next());
+        Assert.assertEquals(100L * (i + 1), resultSet.getLong(1));
+        Assert.assertEquals(startValue + i, resultSet.getLong(2));
+      }
+      Assert.assertFalse(resultSet.next());
+    }
+  }
+
+  private void assertTsFileOrder(final File directory, final String... 
expectedFileNames) {
+    final File[] files = directory.listFiles((dir, name) -> 
name.endsWith(".tsfile"));
+    Assert.assertNotNull(files);
+    Arrays.sort(files, Comparator.comparing(File::getName));
+    Assert.assertEquals(
+        Arrays.asList(expectedFileNames),
+        Arrays.stream(files).map(File::getName).collect(Collectors.toList()));
+  }
+
+  private void assertOneTreeValue(
+      final Statement statement,
+      final String device,
+      final long expectedTime,
+      final long expectedValue)
+      throws SQLException {
+    try (ResultSet resultSet =
+        statement.executeQuery("select value from " + device + " order by 
time")) {
+      Assert.assertTrue(resultSet.next());
+      Assert.assertEquals(expectedTime, resultSet.getLong(1));
+      Assert.assertEquals(expectedValue, resultSet.getLong(2));
+      Assert.assertFalse(resultSet.next());
+    }
+  }
+
+  private void writeTreeFile(
+      final File file, final String device, final long[] timestamps, final 
long[] values)
+      throws Exception {
+    Assert.assertEquals(timestamps.length, values.length);
+    final MeasurementSchema schema = new MeasurementSchema("value", 
TSDataType.INT64);
+    try (TsFileWriter writer = new TsFileWriter(file)) {
+      writer.registerTimeseries(new Path(device), 
Collections.singletonList(schema));
+      final Tablet tablet = new Tablet(device, 
Collections.singletonList(schema));
+      for (int i = 0; i < timestamps.length; i++) {
+        final int row = tablet.getRowSize();
+        tablet.addTimestamp(row, timestamps[i]);
+        tablet.addValue("value", row, values[i]);
+      }
+      writer.writeTree(tablet);
+    }
+  }
+
+  private void writeTypedTreeFile(
+      final File file,
+      final String device,
+      final TSDataType dataType,
+      final long[] timestamps,
+      final Object[] values)
+      throws Exception {
+    Assert.assertEquals(timestamps.length, values.length);
+    final MeasurementSchema schema = new MeasurementSchema("value", dataType);
+    try (TsFileWriter writer = new TsFileWriter(file)) {
+      writer.registerTimeseries(new Path(device), 
Collections.singletonList(schema));
+      final Tablet tablet = new Tablet(device, 
Collections.singletonList(schema));
+      for (int i = 0; i < timestamps.length; i++) {
+        final int row = tablet.getRowSize();
+        tablet.addTimestamp(row, timestamps[i]);
+        tablet.addValue("value", row, values[i]);
+      }
+      writer.writeTree(tablet);
+    }
+  }
+
+  private void writeTreeDeletion(
+      final File file,
+      final String device,
+      final String measurement,
+      final long startTime,
+      final long endTime)
+      throws IOException, IllegalPathException {
+    try (ModificationFile modifications =
+        new ModificationFile(
+            new File(file.getAbsolutePath() + ModificationFile.FILE_SUFFIX), 
false)) {
+      modifications.write(
+          new TreeDeletionEntry(new MeasurementPath(device, measurement), 
startTime, endTime));
+    }
+  }
+
+  private void writeInvalidPathTsFile(final File file, final 
StringArrayDeviceID invalidDevice)
+      throws Exception {
+    final StringArrayDeviceID validDevice =
+        new StringArrayDeviceID(new String[] {"root", "load_p120", 
"valid_device"});
+    try (TsFileIOWriter writer = new TsFileIOWriter(file)) {
+      writer.setSchema(new Schema());
+      writeInvalidPathChunkGroup(writer, validDevice, 1, 7, 42.0);
+      writeInvalidPathChunkGroup(writer, invalidDevice, 2, 1, 1.0);
+      writer.endFile();
+    }
+  }
+
+  private void writeInvalidPathChunkGroup(
+      final TsFileIOWriter writer,
+      final StringArrayDeviceID device,
+      final long timestamp,
+      final int quality,
+      final double value)
+      throws IOException {
+    writer.startChunkGroup(device);
+    final AlignedChunkWriterImpl chunkWriter =
+        new AlignedChunkWriterImpl(
+            List.of(
+                new MeasurementSchema("quality", TSDataType.INT32),
+                new MeasurementSchema("value", TSDataType.DOUBLE)));
+    chunkWriter.getTimeChunkWriter().write(timestamp);
+    chunkWriter.getValueChunkWriterByIndex(0).write(timestamp, quality, false);
+    chunkWriter.getValueChunkWriterByIndex(1).write(timestamp, value, false);
+    chunkWriter.writeToFileWriter(writer);
+    writer.endChunkGroup();
+  }
+
+  private void deleteDatabase(final Statement statement, final String 
database) {
+    try {
+      statement.execute("delete database " + database);
+    } catch (SQLException ignored) {
+      // The database may not have been created by the test.
+    }
+  }
+
+  private long[] buildTimes(final int count, final long timeGap) {
+    final long[] times = new long[count];
+    for (int i = 0; i < count; i++) {
+      times[i] = timeGap * (i + 1);
+    }
+    return times;
+  }
+
+  private long[] buildValues(final int count, final long startValue) {
+    final long[] values = new long[count];
+    for (int i = 0; i < count; i++) {
+      values[i] = startValue + i;
+    }
+    return values;
+  }
+
+  private void deleteRecursively(final File file) throws IOException {
+    if (file == null || !file.exists()) {
+      return;
+    }
+    final File[] children = file.listFiles();
+    if (children != null) {
+      for (final File child : children) {
+        deleteRecursively(child);
+      }
+    }
+    Files.deleteIfExists(file.toPath());
+  }
+}
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
index 551130ad6b9..d7c15da31f6 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
@@ -478,9 +478,24 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
       this.isTableModelTsFileReliableIndex = i;
 
       if (0 <= tabletConversionThresholdBytes
-          && tsFile.length() <= tabletConversionThresholdBytes
-          && handleSingleMiniFile(i)) {
-        return;
+          && tsFile.length() <= tabletConversionThresholdBytes) {
+        if (!isTableModelFile
+            && !isConvertOnTypeMismatch
+            && treeSchemaAutoCreatorAndVerifier != null) {
+          treeSchemaAutoCreatorAndVerifier.flush();
+        }
+        final long startTime = System.nanoTime();
+        try {
+          if (!isTableModelFile) {
+            checkMiniFileTreeWritePermission(tsFile, reader);
+          }
+          if (handleSingleMiniFile(i)) {
+            return;
+          }
+        } finally {
+          LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost(
+              LoadTsFileCostMetricsSet.ANALYSIS_CAST_TABLETS, 
System.nanoTime() - startTime);
+        }
       }
 
       if (isTableModelFile) {
@@ -536,40 +551,67 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
   }
 
   private boolean handleSingleMiniFile(final int i) throws 
FileNotFoundException {
-    final long startTime = System.nanoTime();
-    try {
-      final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
-          new LoadTsFileDataTypeConverter(context, isGeneratedByPipe);
-
-      final TSStatus status =
-          isTableModelTsFile.get(i)
-              ? loadTsFileDataTypeConverter
-                  
.convertForTableModel(createTableConversionStatement(tsFiles.get(i)))
-                  .orElse(null)
-              : loadTsFileDataTypeConverter
-                  
.convertForTreeModel(createTreeConversionStatement(tsFiles.get(i)))
-                  .orElse(null);
-
-      if (status == null || !loadTsFileDataTypeConverter.isSuccessful(status)) 
{
-        LOGGER.warn(
-            DataNodeQueryMessages
-                
.LOAD_FAILED_TO_CONVERT_MINI_TSFILE_ARG_TO_TABLETS_FROM_STATEMENT_ARG_STATUS_ARG,
-            tsFiles.get(i).getPath(),
-            isTableModelStatement ? loadTsFileTableStatement : 
loadTsFileTreeStatement,
-            status);
-        return false;
-      }
+    final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
+        new LoadTsFileDataTypeConverter(context, isGeneratedByPipe);
 
-      // A mark of successful conversion
-      isMiniTsFile.set(i, Boolean.TRUE);
-      isMiniTsFileConverted = true;
+    final TSStatus status =
+        isTableModelTsFile.get(i)
+            ? loadTsFileDataTypeConverter
+                
.convertForTableModel(createTableConversionStatement(tsFiles.get(i)))
+                .orElse(null)
+            : loadTsFileDataTypeConverter
+                
.convertForTreeModel(createTreeConversionStatement(tsFiles.get(i)))
+                .orElse(null);
 
-      addTsFileResource(null);
-      addWritePointCount(0);
-      return true;
-    } finally {
-      LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost(
-          LoadTsFileCostMetricsSet.ANALYSIS_CAST_TABLETS, System.nanoTime() - 
startTime);
+    if (status == null || !loadTsFileDataTypeConverter.isSuccessful(status)) {
+      LOGGER.warn(
+          DataNodeQueryMessages
+              
.LOAD_FAILED_TO_CONVERT_MINI_TSFILE_ARG_TO_TABLETS_FROM_STATEMENT_ARG_STATUS_ARG,
+          tsFiles.get(i).getPath(),
+          isTableModelStatement ? loadTsFileTableStatement : 
loadTsFileTreeStatement,
+          status);
+      return false;
+    }
+
+    // A mark of successful conversion
+    isMiniTsFile.set(i, Boolean.TRUE);
+    isMiniTsFileConverted = true;
+
+    addTsFileResource(null);
+    addWritePointCount(0);
+    return true;
+  }
+
+  private void checkMiniFileTreeWritePermission(
+      final File tsFile, final TsFileSequenceReader reader)
+      throws IOException, LoadAnalyzeException, AuthException {
+    final TsFileResource tsFileResource = constructTsFileResource(reader, 
tsFile);
+    final TreeSchemaAutoCreatorAndVerifier verifier = 
getOrCreateTreeSchemaVerifier();
+    verifier.setCurrentModificationsAndTimeIndex(tsFileResource, reader);
+
+    final TsFileSequenceReaderTimeseriesMetadataIterator 
timeseriesMetadataIterator =
+        new TsFileSequenceReaderTimeseriesMetadataIterator(
+            reader,
+            true,
+            IoTDBDescriptor.getInstance()
+                .getConfig()
+                .getLoadTsFileAnalyzeSchemaBatchReadTimeSeriesMetadataCount());
+    while (timeseriesMetadataIterator.hasNext()) {
+      final Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadata 
=
+          timeseriesMetadataIterator.next();
+      TsFileResourceUtils.updateTsFileResource(
+          device2TimeseriesMetadata,
+          tsFileResource,
+          
IoTDBDescriptor.getInstance().getConfig().isCacheLastValuesForLoad());
+      verifier.setCurrentTimeIndex(tsFileResource.getTimeIndex());
+      for (final Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry :
+          device2TimeseriesMetadata.entrySet()) {
+        try {
+          
verifier.checkWritePermission(Collections.singletonMap(entry.getKey(), 
entry.getValue()));
+        } catch (LoadAnalyzeInvalidPathException ignored) {
+          // Let conversion report invalid paths in file order, after 
processing earlier tablets.
+        }
+      }
     }
   }
 
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
index 3b16720b4a4..220c17f9db8 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
@@ -335,7 +335,7 @@ public class LoadTsFileDataTypeConverter {
             new UserEntity(session.getUserId(), session.getUsername(), 
session.getClientAddress()),
             ZoneId.systemDefault(),
             session.getClientVersion(),
-            session.getDatabaseName(),
+            null,
             SqlDialect.TREE,
             allowAutoCreateSchema
                 && session.isEnableAutoCreateSchema()

Reply via email to