This is an automated email from the ASF dual-hosted git repository.

Caideyipi pushed a commit to branch fix/load-mini-tsfile-20261118
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/fix/load-mini-tsfile-20261118 
by this push:
     new 5d4c5399d24 test(load): cover Tree mini-file database semantics
5d4c5399d24 is described below

commit 5d4c5399d242f5062b0f28880f8ee92ac9a14ae5
Author: Caideyipi <[email protected]>
AuthorDate: Wed Nov 18 17:53:55 2026 +0800

    test(load): cover Tree mini-file database semantics
---
 .../relational/it/db/it/IoTDBLoadTsFileMiniIT.java | 381 +++++++++++++++++++++
 1 file changed, 381 insertions(+)

diff --git 
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileMiniIT.java
 
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileMiniIT.java
new file mode 100644
index 00000000000..1fdd3ef92bd
--- /dev/null
+++ 
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileMiniIT.java
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.relational.it.db.it;
+
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBLoadTsFileMiniIT {
+  private static final String TABLE_DATABASE = "load_p116_target";
+  private static final String SESSION_DATABASE = "load_p116_session";
+  private static final String TREE_DATABASE = "root.load_p116_tree";
+
+  private File tmpDir;
+
+  @BeforeClass
+  public static void setUpClass() throws Exception {
+    
EnvFactory.getEnv().getConfig().getCommonConfig().setTimePartitionInterval(10_000L);
+    
EnvFactory.getEnv().getConfig().getCommonConfig().setEnforceStrongPassword(false);
+    EnvFactory.getEnv()
+        .getConfig()
+        .getCommonConfig()
+        .setAutoCreateSchemaEnabled(true)
+        .setDatanodeMemoryProportion("3:3:1:1:1:0");
+    EnvFactory.getEnv()
+        .getConfig()
+        .getDataNodeConfig()
+        .setMetricReporterType(Collections.singletonList("PROMETHEUS"))
+        .setMetricPrometheusReporterAsyncUpdate(false);
+    EnvFactory.getEnv()
+        .getConfig()
+        .getConfigNodeConfig()
+        .setMetricReporterType(Collections.singletonList("PROMETHEUS"));
+    EnvFactory.getEnv().initClusterEnvironment();
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    tmpDir = Files.createTempDirectory("load-mini").toFile();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    try (Connection connection = 
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+        Statement statement = connection.createStatement()) {
+      statement.execute("drop database if exists " + TABLE_DATABASE);
+      statement.execute("drop database if exists " + SESSION_DATABASE);
+      try (Connection treeConnection = EnvFactory.getEnv().getConnection();
+          Statement treeStatement = treeConnection.createStatement()) {
+        try {
+          treeStatement.execute("delete database " + TREE_DATABASE);
+        } catch (SQLException ignored) {
+          // The table-only case does not create the Tree database.
+        }
+      }
+    } finally {
+      deleteRecursively(tmpDir);
+    }
+  }
+
+  @AfterClass
+  public static void tearDownClass() {
+    EnvFactory.getEnv().cleanClusterEnvironment();
+  }
+
+  @Test
+  public void testTableMiniConversionHonorsExplicitAndSessionDatabases() 
throws Exception {
+    final File explicitFile = new File(tmpDir, "mini-explicit.tsfile");
+    final File sessionFile = new File(tmpDir, "mini-session.tsfile");
+    writeMiniTableFile(explicitFile, "mini_explicit", true);
+    writeMiniTableFile(sessionFile, "mini_session", false);
+    final long threshold = Math.max(explicitFile.length(), 
sessionFile.length());
+
+    try (Connection connection = 
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+        Statement statement = connection.createStatement()) {
+      statement.execute("create database " + TABLE_DATABASE);
+      statement.execute("create database " + SESSION_DATABASE);
+      statement.execute("use " + TABLE_DATABASE);
+      statement.execute(
+          "create table mini_explicit (site STRING TAG, zone STRING TAG, 
reading INT32 FIELD)");
+      statement.execute("use " + SESSION_DATABASE);
+      statement.execute(
+          "create table mini_session (site STRING TAG, zone STRING TAG, 
reading INT32 FIELD)");
+    }
+
+    final long conversionsBefore = getMiniConversionCount();
+    try (Connection connection = 
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+        Statement statement = connection.createStatement()) {
+      statement.execute("use " + SESSION_DATABASE);
+      statement.execute(
+          String.format(
+              "load '%s' with ('database'='%s','database-level'='2',"
+                  + 
"'tablet-conversion-threshold'='%d','auto-create-schema'='false',"
+                  + "'on-success'='none')",
+              explicitFile.getAbsolutePath(), TABLE_DATABASE, threshold));
+      try (ResultSet resultSet =
+          statement.executeQuery("select time, site, zone, reading from 
mini_session")) {
+        Assert.assertFalse("Explicit database LOAD changed the session 
database", resultSet.next());
+      }
+
+      statement.execute(
+          String.format(
+              "load '%s' with 
('database-level'='2','tablet-conversion-threshold'='%d',"
+                  + "'auto-create-schema'='false','on-success'='none')",
+              sessionFile.getAbsolutePath(), threshold));
+      try (ResultSet resultSet =
+          statement.executeQuery("select time, site, zone, reading from 
mini_session")) {
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(20, resultSet.getLong("time"));
+        Assert.assertEquals("east", resultSet.getString("site"));
+        Assert.assertNull(resultSet.getString("zone"));
+        Assert.assertEquals(9, resultSet.getInt("reading"));
+        Assert.assertFalse(resultSet.next());
+      }
+      try (ResultSet resultSet = statement.executeQuery("show tables")) {
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals("mini_session", resultSet.getString(1));
+        Assert.assertFalse(resultSet.next());
+      }
+    }
+
+    try (Connection connection = 
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+        Statement statement = connection.createStatement()) {
+      statement.execute("use " + TABLE_DATABASE);
+      try (ResultSet resultSet =
+          statement.executeQuery("select time, site, zone, reading from 
mini_explicit")) {
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(11, resultSet.getLong("time"));
+        Assert.assertEquals("south", resultSet.getString("site"));
+        Assert.assertEquals("south-1", resultSet.getString("zone"));
+        Assert.assertEquals(8, resultSet.getInt("reading"));
+        Assert.assertFalse(resultSet.next());
+      }
+    }
+    Assert.assertEquals(2, getMiniConversionCount() - conversionsBefore);
+    Assert.assertTrue(explicitFile.exists());
+    Assert.assertTrue(sessionFile.exists());
+  }
+
+  @Test
+  public void testMixedMiniAndNormalFilesKeepModelState() throws Exception {
+    final File loadDir = new File(tmpDir, "mixed-mini");
+    Assert.assertTrue(loadDir.mkdir());
+    final File firstMini = new File(loadDir, "01-tree-mini.tsfile");
+    final File normalTable = new File(loadDir, "02-table-normal.tsfile");
+    final File lastMini = new File(loadDir, "03-tree-mini.tsfile");
+    writeTreeFile(firstMini, TREE_DATABASE + ".before", 1, 101);
+    writeMiniTableFile(normalTable, "normal_table", true);
+    writeTreeFile(lastMini, TREE_DATABASE + ".after", 3, 303);
+    assertTsFileOrder(
+        loadDir, "01-tree-mini.tsfile", "02-table-normal.tsfile", 
"03-tree-mini.tsfile");
+    final long threshold = Math.max(firstMini.length(), lastMini.length());
+    Assert.assertTrue(normalTable.length() > threshold);
+    createTreeSchema("before", "after");
+    final long conversionsBefore = getMiniConversionCount();
+    try (Connection connection = 
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+        Statement statement = connection.createStatement()) {
+      statement.execute("create database " + TABLE_DATABASE);
+      statement.execute("create database " + SESSION_DATABASE);
+      statement.execute("use " + TABLE_DATABASE);
+      statement.execute(
+          "create table normal_table (site STRING TAG, zone STRING TAG, 
reading INT32 FIELD)");
+      statement.execute("use " + SESSION_DATABASE);
+      statement.execute(
+          String.format(
+              "load '%s' with ('database'='%s','database-level'='2',"
+                  + 
"'tablet-conversion-threshold'='%d','auto-create-schema'='false',"
+                  + "'convert-on-type-mismatch'='false','on-success'='none')",
+              loadDir.getAbsolutePath(), TABLE_DATABASE, threshold));
+      statement.execute("create table session_probe (site STRING TAG, reading 
INT32 FIELD)");
+      statement.execute("insert into session_probe(time, site, reading) values 
(20, 'session', 2)");
+      try (ResultSet resultSet = statement.executeQuery("select reading from 
session_probe")) {
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(2, resultSet.getInt(1));
+        Assert.assertFalse(resultSet.next());
+      }
+      statement.execute("use " + TABLE_DATABASE);
+      try (ResultSet resultSet =
+          statement.executeQuery("select time, site, zone, reading from 
normal_table")) {
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(11, resultSet.getLong("time"));
+        Assert.assertEquals("south", resultSet.getString("site"));
+        Assert.assertEquals("south-1", resultSet.getString("zone"));
+        Assert.assertEquals(8, resultSet.getInt("reading"));
+        Assert.assertFalse(resultSet.next());
+      }
+    }
+    Assert.assertEquals(2, getMiniConversionCount() - conversionsBefore);
+    assertTreeValue(TREE_DATABASE + ".before", 1, 101);
+    assertTreeValue(TREE_DATABASE + ".after", 3, 303);
+    Assert.assertTrue(firstMini.exists());
+    Assert.assertTrue(normalTable.exists());
+    Assert.assertTrue(lastMini.exists());
+  }
+
+  private void writeMiniTableFile(
+      final File file, final String tableName, final boolean withDeletion) 
throws Exception {
+    final List<IMeasurementSchema> schemas =
+        Arrays.asList(
+            new MeasurementSchema("site", TSDataType.STRING),
+            new MeasurementSchema("zone", TSDataType.STRING),
+            new MeasurementSchema("reading", TSDataType.INT32));
+    final List<ColumnCategory> categories =
+        Arrays.asList(ColumnCategory.TAG, ColumnCategory.TAG, 
ColumnCategory.FIELD);
+    try (TsFileWriter writer = new TsFileWriter(file)) {
+      writeTable(
+          writer,
+          tableName,
+          schemas,
+          categories,
+          withDeletion
+              ? Arrays.asList(
+                  new Object[] {10L, "north", null, 7}, new Object[] {11L, 
"south", "south-1", 8})
+              : Collections.singletonList(new Object[] {20L, "east", null, 
9}));
+    }
+    if (withDeletion) {
+      try (ModificationFile modifications =
+          new ModificationFile(
+              new File(file.getAbsolutePath() + ModificationFile.FILE_SUFFIX), 
false)) {
+        modifications.write(
+            new TableDeletionEntry(new DeletionPredicate(tableName), new 
TimeRange(10, 10)));
+      }
+    }
+  }
+
+  private long getMiniConversionCount() {
+    long count = 0;
+    for (final String content : 
EnvFactory.getEnv().getMetricPrometheusReporterContents(null)) {
+      if (content == null) {
+        continue;
+      }
+      for (final String line : content.split("\\n")) {
+        final int labelStart = line.indexOf('{');
+        if (labelStart < 0
+            || !line.substring(0, labelStart).endsWith("_count")
+            || !line.contains("load_time_cost")
+            || !line.contains("analysis_cast_tablets")) {
+          continue;
+        }
+        count += 
Math.round(Double.parseDouble(line.substring(line.lastIndexOf(' ') + 
1).trim()));
+      }
+    }
+    return count;
+  }
+
+  private void createTreeSchema(final String... devices) throws SQLException {
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement()) {
+      statement.execute("create database " + TREE_DATABASE);
+      for (final String device : devices) {
+        statement.execute("create timeseries " + TREE_DATABASE + "." + device 
+ ".value INT64");
+      }
+    }
+  }
+
+  private void writeTable(
+      final TsFileWriter writer,
+      final String tableName,
+      final List<IMeasurementSchema> schemas,
+      final List<ColumnCategory> categories,
+      final List<Object[]> rows)
+      throws IOException, WriteProcessException {
+    writer.registerTableSchema(new TableSchema(tableName, schemas, 
categories));
+    final List<String> names =
+        
schemas.stream().map(IMeasurementSchema::getMeasurementName).collect(Collectors.toList());
+    final List<TSDataType> types =
+        
schemas.stream().map(IMeasurementSchema::getType).collect(Collectors.toList());
+    final Tablet tablet = new Tablet(tableName, names, types, categories);
+    for (final Object[] values : rows) {
+      final int row = tablet.getRowSize();
+      tablet.addTimestamp(row, (Long) values[0]);
+      for (int column = 0; column < schemas.size(); column++) {
+        tablet.addValue(names.get(column), row, values[column + 1]);
+      }
+    }
+    writer.writeTable(tablet);
+  }
+
+  private void assertTsFileOrder(final File directory, final String... 
expectedNames) {
+    final File[] files = directory.listFiles((dir, name) -> 
name.endsWith(".tsfile"));
+    Assert.assertNotNull(files);
+    Arrays.sort(files, Comparator.comparing(File::getName));
+    Assert.assertEquals(
+        Arrays.asList(expectedNames),
+        Arrays.stream(files).map(File::getName).collect(Collectors.toList()));
+  }
+
+  private void writeTreeFile(
+      final File file, final String device, final long timestamp, final long 
value)
+      throws Exception {
+    final MeasurementSchema schema = new MeasurementSchema("value", 
TSDataType.INT64);
+    try (TsFileWriter writer = new TsFileWriter(file)) {
+      writer.registerTimeseries(new Path(device), 
Collections.singletonList(schema));
+      final Tablet tablet = new Tablet(device, 
Collections.singletonList(schema));
+      tablet.addTimestamp(0, timestamp);
+      tablet.addValue(0, 0, value);
+      writer.writeTree(tablet);
+    }
+  }
+
+  private void assertTreeValue(
+      final String device, final long expectedTime, final long expectedValue) 
throws SQLException {
+    try (Connection connection = EnvFactory.getEnv().getConnection();
+        Statement statement = connection.createStatement();
+        ResultSet resultSet =
+            statement.executeQuery("select value from " + device + " order by 
time")) {
+      Assert.assertTrue(resultSet.next());
+      Assert.assertEquals(expectedTime, resultSet.getLong(1));
+      Assert.assertEquals(expectedValue, resultSet.getLong(2));
+      Assert.assertFalse(resultSet.next());
+    }
+  }
+
+  private void deleteRecursively(final File file) throws IOException {
+    if (file == null || !file.exists()) {
+      return;
+    }
+    final File[] children = file.listFiles();
+    if (children != null) {
+      for (final File child : children) {
+        deleteRecursively(child);
+      }
+    }
+    Files.deleteIfExists(file.toPath());
+  }
+}

Reply via email to