This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 229afb5b922 fix(load): validate Tree mini-file permissions before
conversion (#19317)
229afb5b922 is described below
commit 229afb5b9221141abd7f996f1335a98b3ea53cbc
Author: Caideyipi <[email protected]>
AuthorDate: Wed Nov 18 18:48:48 2026 +0800
fix(load): validate Tree mini-file permissions before conversion (#19317)
* fix(load): validate tree mini-file permissions before conversion
* test(load): cover Tree mini-file database semantics
---
.../it/env/cluster/config/MppDataNodeConfig.java | 7 +
.../it/env/remote/config/RemoteDataNodeConfig.java | 5 +
.../apache/iotdb/itbase/env/DataNodeConfig.java | 2 +
.../iotdb/db/it/IoTDBLoadTsFileBoundaryIT.java | 769 +++++++++++++++++++++
.../relational/it/db/it/IoTDBLoadTsFileMiniIT.java | 381 ++++++++++
.../plan/analyze/load/LoadTsFileAnalyzer.java | 112 ++-
.../converter/LoadTsFileDataTypeConverter.java | 2 +-
7 files changed, 1242 insertions(+), 36 deletions(-)
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java
index 95efb88b754..a15feddb4e0 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppDataNodeConfig.java
@@ -56,6 +56,13 @@ public class MppDataNodeConfig extends MppBaseConfig
implements DataNodeConfig {
return this;
}
+ @Override
+ public DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean
asyncUpdate) {
+ properties.setProperty(
+ "dn_metric_prometheus_reporter_async_update",
String.valueOf(asyncUpdate));
+ return this;
+ }
+
@Override
public DataNodeConfig setMetricPrometheusReporterUsername(String username) {
properties.setProperty("metric_prometheus_reporter_username", username);
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java
index d8350577f57..a1db5cfc04e 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java
@@ -28,6 +28,11 @@ public class RemoteDataNodeConfig implements DataNodeConfig {
return this;
}
+ @Override
+ public DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean
asyncUpdate) {
+ return this;
+ }
+
@Override
public DataNodeConfig setMetricPrometheusReporterUsername(String username) {
return this;
diff --git
a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java
b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java
index 99c360d2678..183eadcac8b 100644
---
a/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java
+++
b/integration-test/src/main/java/org/apache/iotdb/itbase/env/DataNodeConfig.java
@@ -25,6 +25,8 @@ import java.util.List;
public interface DataNodeConfig {
DataNodeConfig setMetricReporterType(List<String> metricReporterTypes);
+ DataNodeConfig setMetricPrometheusReporterAsyncUpdate(boolean asyncUpdate);
+
DataNodeConfig setMetricPrometheusReporterUsername(String username);
DataNodeConfig setMetricPrometheusReporterPassword(String password);
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileBoundaryIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileBoundaryIT.java
new file mode 100644
index 00000000000..10710ef6400
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileBoundaryIT.java
@@ -0,0 +1,769 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.db.it;
+
+import org.apache.iotdb.commons.auth.entity.PrivilegeType;
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.iotdb.commons.path.PartialPath;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.StringArrayDeviceID;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.schema.Schema;
+import org.apache.tsfile.write.writer.TsFileIOWriter;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.apache.iotdb.db.it.utils.TestUtils.createUser;
+import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSeriesPrivilege;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({LocalStandaloneIT.class, ClusterIT.class})
+public class IoTDBLoadTsFileBoundaryIT {
+ private static final String MINI_DATABASE = "root.load_p115";
+ private static final String MIXED_DATABASE = "root.load_p115_mixed";
+ private static final String ALL_MINI_DATABASE = "root.load_p115_all_mini";
+ private static final String INVALID_DATABASE = "root.load_p120";
+
+ private File tmpDir;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+
EnvFactory.getEnv().getConfig().getCommonConfig().setTimePartitionInterval(10_000L);
+
EnvFactory.getEnv().getConfig().getCommonConfig().setEnforceStrongPassword(false);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getConfigNodeConfig()
+ .setMetricReporterType(Collections.singletonList("PROMETHEUS"));
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeConfig()
+ .setMetricReporterType(Collections.singletonList("PROMETHEUS"))
+ .setMetricPrometheusReporterAsyncUpdate(false);
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ tmpDir = Files.createTempDirectory("load-boundary").toFile();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ deleteDatabase(statement, MINI_DATABASE);
+ for (int i = 0; i < 4; i++) {
+ deleteDatabase(statement, MINI_DATABASE + "_threshold_" + i);
+ }
+ deleteDatabase(statement, MIXED_DATABASE);
+ deleteDatabase(statement, ALL_MINI_DATABASE);
+ deleteDatabase(statement, INVALID_DATABASE);
+ } catch (SQLException ignored) {
+ // The cluster may already be unavailable after a failed test.
+ } finally {
+ deleteRecursively(tmpDir);
+ }
+ }
+
+ @AfterClass
+ public static void tearDownClass() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testMiniFileThresholdAndMixedStatementReconstruction() throws
Exception {
+ for (int thresholdIndex = 0; thresholdIndex < 4; thresholdIndex++) {
+ final String thresholdDatabase = MINI_DATABASE + "_threshold_" +
thresholdIndex;
+ final String thresholdDevice = thresholdDatabase + ".threshold_device";
+ final File thresholdDir = new File(tmpDir, "threshold-" +
thresholdIndex);
+ Assert.assertTrue(thresholdDir.mkdir());
+ final File thresholdFile = new File(thresholdDir, "threshold.tsfile");
+ writeTreeFile(
+ thresholdFile, thresholdDevice, new long[] {9999, 10000, 20000}, new
long[] {9, 10, 20});
+ writeTreeDeletion(thresholdFile, thresholdDevice, "value", 9999, 9999);
+ final long fileSize = thresholdFile.length();
+ final long threshold = thresholdIndex == 0 ? -1 : fileSize +
thresholdIndex - 2;
+
+ final long conversionsBefore = getMiniConversionCount();
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database " + thresholdDatabase);
+ statement.execute("create timeseries " + thresholdDevice + ".value
INT64");
+ statement.execute(
+ String.format(
+ "load \"%s\" with
('database-level'='2','auto-create-schema'='false',"
+ +
"'on-success'='none','tablet-conversion-threshold'='%d')",
+ thresholdFile.getAbsolutePath(), threshold));
+ assertThresholdFileContents(statement, thresholdDevice);
+ }
+
+ final long expectedConversionDelta = thresholdIndex >= 2 ? 1 : 0;
+ Assert.assertEquals(
+ "The load_time_cost analysis_cast_tablets metric must prove the
selected path",
+ expectedConversionDelta,
+ getMiniConversionCount() - conversionsBefore);
+ Assert.assertTrue(thresholdFile.exists());
+ Assert.assertTrue(
+ new File(thresholdFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX).exists());
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ deleteDatabase(statement, thresholdDatabase);
+ }
+ }
+
+ final File deleteOnSuccessFile = new File(tmpDir,
"threshold-delete-on-success.tsfile");
+ final String deleteOnSuccessDevice = MINI_DATABASE + ".delete_on_success";
+ writeTreeFile(
+ deleteOnSuccessFile,
+ deleteOnSuccessDevice,
+ new long[] {9999, 10000, 20000},
+ new long[] {9, 10, 20});
+ writeTreeDeletion(deleteOnSuccessFile, deleteOnSuccessDevice, "value",
9999, 9999);
+ final long deleteConversionBefore = getMiniConversionCount();
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database " + MINI_DATABASE);
+ statement.execute("create timeseries " + deleteOnSuccessDevice + ".value
INT64");
+ statement.execute(
+ String.format(
+ "load \"%s\" with
('database-level'='2','auto-create-schema'='false',"
+ +
"'on-success'='delete','tablet-conversion-threshold'='%d')",
+ deleteOnSuccessFile.getAbsolutePath(),
deleteOnSuccessFile.length()));
+ assertThresholdFileContents(statement, deleteOnSuccessDevice);
+ }
+ Assert.assertEquals(1, getMiniConversionCount() - deleteConversionBefore);
+ Assert.assertFalse(deleteOnSuccessFile.exists());
+ Assert.assertFalse(
+ new File(deleteOnSuccessFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX).exists());
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ deleteDatabase(statement, MINI_DATABASE);
+ }
+
+ final File mixedDir = new File(tmpDir, "mixed");
+ Assert.assertTrue(mixedDir.mkdir());
+ final File firstMini = new File(mixedDir, "01-first-mini.tsfile");
+ final File firstNormal = new File(mixedDir, "02-first-normal.tsfile");
+ final File middleMini = new File(mixedDir, "03-middle-mini.tsfile");
+ final File lastNormal = new File(mixedDir, "04-last-normal.tsfile");
+ final File lastMini = new File(mixedDir, "05-last-mini.tsfile");
+ writeTreeFile(firstMini, MIXED_DATABASE + ".small", new long[] {1}, new
long[] {11});
+ writeTreeFile(
+ firstNormal, MIXED_DATABASE + ".normal_first", buildTimes(64, 100),
buildValues(64, 1000));
+ writeTreeFile(middleMini, MIXED_DATABASE + ".middle", new long[] {2}, new
long[] {22});
+ writeTreeFile(
+ lastNormal, MIXED_DATABASE + ".normal_last", buildTimes(64, 100),
buildValues(64, 2000));
+ writeTreeFile(lastMini, MIXED_DATABASE + ".tail", new long[] {2}, new
long[] {33});
+
+ assertTsFileOrder(
+ mixedDir,
+ "01-first-mini.tsfile",
+ "02-first-normal.tsfile",
+ "03-middle-mini.tsfile",
+ "04-last-normal.tsfile",
+ "05-last-mini.tsfile");
+ final long threshold =
+ Math.max(Math.max(firstMini.length(), middleMini.length()),
lastMini.length());
+ Assert.assertTrue(
+ "The normal files must remain above the conversion threshold",
+ firstNormal.length() > threshold && lastNormal.length() > threshold);
+ final long conversionsBefore = getMiniConversionCount();
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database " + MIXED_DATABASE);
+ for (final String device :
+ List.of(
+ MIXED_DATABASE + ".small",
+ MIXED_DATABASE + ".normal_first",
+ MIXED_DATABASE + ".middle",
+ MIXED_DATABASE + ".normal_last",
+ MIXED_DATABASE + ".tail")) {
+ statement.execute("create timeseries " + device + ".value INT64");
+ }
+ statement.execute(buildLoadSql(mixedDir, threshold, "2", "none"));
+ assertOneTreeValue(statement, MIXED_DATABASE + ".small", 1, 11);
+ assertOneTreeValue(statement, MIXED_DATABASE + ".middle", 2, 22);
+ assertOneTreeValue(statement, MIXED_DATABASE + ".tail", 2, 33);
+ assertTreeSeriesValues(statement, MIXED_DATABASE + ".normal_first", 64,
1000);
+ assertTreeSeriesValues(statement, MIXED_DATABASE + ".normal_last", 64,
2000);
+ }
+ Assert.assertEquals(3, getMiniConversionCount() - conversionsBefore);
+ Assert.assertTrue(firstMini.exists());
+ Assert.assertTrue(firstNormal.exists());
+ Assert.assertTrue(middleMini.exists());
+ Assert.assertTrue(lastNormal.exists());
+ Assert.assertTrue(lastMini.exists());
+
+ final File allMiniDir = new File(tmpDir, "all-mini");
+ Assert.assertTrue(allMiniDir.mkdir());
+ final File[] allMiniFiles = {
+ new File(allMiniDir, "01-one.tsfile"),
+ new File(allMiniDir, "02-two.tsfile"),
+ new File(allMiniDir, "03-three.tsfile")
+ };
+ for (int i = 0; i < allMiniFiles.length; i++) {
+ writeTreeFile(
+ allMiniFiles[i],
+ ALL_MINI_DATABASE + ".device_" + i,
+ new long[] {i + 1},
+ new long[] {100 + i});
+ }
+ assertTsFileOrder(allMiniDir, "01-one.tsfile", "02-two.tsfile",
"03-three.tsfile");
+ final long allMiniThreshold =
+
Arrays.stream(allMiniFiles).mapToLong(File::length).max().orElseThrow();
+ final long allMiniConversionsBefore = getMiniConversionCount();
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database " + ALL_MINI_DATABASE);
+ for (int i = 0; i < allMiniFiles.length; i++) {
+ statement.execute(
+ "create timeseries " + ALL_MINI_DATABASE + ".device_" + i +
".value INT64");
+ }
+ statement.execute(buildLoadSql(allMiniDir, allMiniThreshold, "2",
"none"));
+ for (int i = 0; i < allMiniFiles.length; i++) {
+ assertOneTreeValue(statement, ALL_MINI_DATABASE + ".device_" + i, i +
1, 100 + i);
+ Assert.assertTrue(allMiniFiles[i].exists());
+ }
+ }
+ Assert.assertEquals(3, getMiniConversionCount() -
allMiniConversionsBefore);
+ }
+
+ @Test
+ public void testMiniFileSuccessDoesNotHideFollowingFailure() throws
Exception {
+ final File loadDir = new File(tmpDir, "mini-before-failure");
+ Assert.assertTrue(loadDir.mkdir());
+ final String smallDevice = MIXED_DATABASE + ".small_before_failure";
+ final String failingDevice = MIXED_DATABASE + ".schema_mismatch";
+ final String unprocessedDevice = MIXED_DATABASE + ".mini_after_failure";
+ final File firstMini = new File(loadDir, "01-mini.tsfile");
+ final File failingNormal = new File(loadDir, "02-normal-failure.tsfile");
+ final File tailMini = new File(loadDir, "03-tail-mini.tsfile");
+ writeTreeFile(firstMini, smallDevice, new long[] {1}, new long[] {11});
+ final long[] failingTimes = buildTimes(256, 10);
+ final Object[] failingValues = new Object[failingTimes.length];
+ final Random random = new Random(0);
+ for (int i = 0; i < failingValues.length; i++) {
+ failingValues[i] = random.nextDouble();
+ }
+ writeTypedTreeFile(
+ failingNormal, failingDevice, TSDataType.DOUBLE, failingTimes,
failingValues);
+ writeTreeFile(tailMini, unprocessedDevice, new long[] {2}, new long[]
{22});
+ assertTsFileOrder(loadDir, "01-mini.tsfile", "02-normal-failure.tsfile",
"03-tail-mini.tsfile");
+
+ final long threshold = Math.max(firstMini.length(), tailMini.length());
+ Assert.assertTrue(
+ String.format(
+ "Normal failure file (%d bytes) must exceed mini threshold (%d
bytes)",
+ failingNormal.length(), threshold),
+ failingNormal.length() > threshold);
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database " + MIXED_DATABASE);
+ statement.execute("create timeseries " + smallDevice + ".value INT64");
+ statement.execute("create timeseries " + failingDevice + ".value INT32");
+ statement.execute("create timeseries " + unprocessedDevice + ".value
INT64");
+ }
+
+ final long conversionsBefore = getMiniConversionCount();
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ try {
+ statement.execute(
+ String.format(
+ "load \"%s\" with
('database-level'='1','tablet-conversion-threshold'='%d',"
+ +
"'auto-create-schema'='false','convert-on-type-mismatch'='false',"
+ + "'on-success'='none')",
+ loadDir.getAbsolutePath(), threshold));
+ Assert.fail("The later type mismatch must fail the whole LOAD
statement");
+ } catch (SQLException expected) {
+ Assert.assertTrue(
+ "The type mismatch for the middle file must be the reported
failure: "
+ + expected.getMessage(),
+ expected.getMessage().contains(failingDevice));
+ }
+ assertOneTreeValue(statement, smallDevice, 1, 11);
+ try (ResultSet resultSet =
+ statement.executeQuery("select count(value) from " + failingDevice))
{
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(0, resultSet.getLong(1));
+ }
+ try (ResultSet resultSet =
+ statement.executeQuery("select count(value) from " +
unprocessedDevice)) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(0, resultSet.getLong(1));
+ }
+ }
+ Assert.assertEquals(1, getMiniConversionCount() - conversionsBefore);
+ Assert.assertTrue("on-success=none must retain the successful mini file",
firstMini.exists());
+ Assert.assertTrue("The failed source file must be retained",
failingNormal.exists());
+ Assert.assertTrue("Files after the failed entry must remain unprocessed",
tailMini.exists());
+ }
+
+ @Test
+ public void testMiniFileChecksSubmittingUser() throws Exception {
+ final String device = MINI_DATABASE + ".permission_probe";
+ final String user = "load_p115_user";
+ final String password = "test123123456";
+ final File tsFile = new File(tmpDir, "mini-user.tsfile");
+ writeTreeFile(tsFile, device, new long[] {1}, new long[] {42});
+ writeTreeDeletion(tsFile, device, "value", 2, 2);
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database " + MINI_DATABASE);
+ statement.execute("create timeseries " + device + ".value INT64");
+ createUser(user, password);
+ try {
+ final String loadSql =
+ String.format(
+ "load \"%s\" with
('auto-create-schema'='false','verify'='false',"
+ +
"'on-success'='delete','tablet-conversion-threshold'='%d')",
+ tsFile.getAbsolutePath(), tsFile.length());
+ final long conversionsBefore = getMiniConversionCount();
+ try (Connection userConnection =
EnvFactory.getEnv().getConnection(user, password);
+ Statement userStatement = userConnection.createStatement()) {
+ try {
+ userStatement.execute(loadSql);
+ Assert.fail("Mini-file conversion must check the submitting user's
WRITE_DATA");
+ } catch (SQLException expected) {
+ Assert.assertEquals(
+ TSStatusCode.NO_PERMISSION.getStatusCode(),
expected.getErrorCode());
+ }
+ }
+ Assert.assertEquals(1, getMiniConversionCount() - conversionsBefore);
+ Assert.assertTrue(tsFile.exists());
+ Assert.assertTrue(
+ new File(tsFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX).exists());
+ try (ResultSet resultSet = statement.executeQuery("select count(value)
from " + device)) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(0, resultSet.getLong(1));
+ }
+ grantUserSeriesPrivilege(user, PrivilegeType.WRITE_DATA, MINI_DATABASE
+ ".**");
+ try (Connection userConnection =
EnvFactory.getEnv().getConnection(user, password);
+ Statement userStatement = userConnection.createStatement()) {
+ userStatement.execute(loadSql);
+ }
+ Assert.assertEquals(2, getMiniConversionCount() - conversionsBefore);
+ assertOneTreeValue(statement, device, 1, 42);
+ Assert.assertFalse(tsFile.exists());
+ Assert.assertFalse(
+ new File(tsFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX).exists());
+ } finally {
+ statement.execute("drop user " + user);
+ }
+ }
+ }
+
+ @Test
+ public void testInvalidTreeDevicePathFailsWithAndWithoutMiniConversion()
throws Exception {
+ final long loadMemoryBefore = getLoadTsFileUsedMemory();
+ for (final boolean nullNode : new boolean[] {false, true}) {
+ final StringArrayDeviceID invalidDevice =
+ new StringArrayDeviceID(
+ nullNode
+ ? new String[] {"root.load_p120", null, "invalid_device"}
+ : new String[] {"root", "load_p120", ""});
+ final File invalidFile = new File(tmpDir, "invalid-path-" + nullNode +
".tsfile");
+ writeInvalidPathTsFile(invalidFile, invalidDevice);
+ writeTreeDeletion(invalidFile, INVALID_DATABASE + ".valid_device",
"value", 1, 1);
+
+ for (final boolean miniConversion : new boolean[] {false, true}) {
+ final File inputDir =
+ new File(tmpDir, (miniConversion ? "invalid-mini-" :
"invalid-normal-") + nullNode);
+ Assert.assertTrue(inputDir.mkdir());
+ final File inputFile = new File(inputDir, "invalid-path.tsfile");
+ Files.copy(invalidFile.toPath(), inputFile.toPath());
+ Files.copy(
+ new File(invalidFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX).toPath(),
+ new File(inputFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX).toPath());
+
+ final long conversionsBefore = getMiniConversionCount();
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ deleteDatabase(statement, INVALID_DATABASE);
+ statement.execute("create database " + INVALID_DATABASE);
+ statement.execute(
+ "create aligned timeseries "
+ + INVALID_DATABASE
+ + ".valid_device(quality INT32, value DOUBLE)");
+ final String attributes =
+ miniConversion
+ ?
"'verify'='false','auto-create-schema'='false','on-success'='delete',"
+ + "'tablet-conversion-threshold'='"
+ + inputFile.length()
+ + "'"
+ :
"'verify'='false','auto-create-schema'='false','on-success'='delete'";
+ try {
+ statement.execute(
+ String.format("load \"%s\" with (%s)",
inputFile.getAbsolutePath(), attributes));
+ Assert.fail("A TsFile with an empty Tree device path must fail
LOAD");
+ } catch (SQLException expected) {
+ Assert.assertTrue(
+ "The error must identify the illegal empty node in the TsFile
device path",
+ expected
+ .getMessage()
+ .contains(
+ new IllegalPathException(new
PartialPath(invalidDevice).getFullPath())
+ .getMessage()));
+ Assert.assertEquals(
+ TSStatusCode.LOAD_FILE_ERROR.getStatusCode(),
expected.getErrorCode());
+ }
+ Assert.assertEquals(
+ "Only the mini-file case should enter tablet-conversion
analysis",
+ miniConversion ? 1 : 0,
+ getMiniConversionCount() - conversionsBefore);
+
+ Assert.assertTrue("Failed LOAD must retain its source TsFile",
inputFile.exists());
+ Assert.assertTrue(
+ "Failed LOAD must retain the source mods",
+ new File(inputFile.getAbsolutePath() +
ModificationFile.FILE_SUFFIX).exists());
+ try (ResultSet resultSet =
+ statement.executeQuery("show timeseries " + INVALID_DATABASE +
".**")) {
+ final Set<String> actualTimeseries = new HashSet<>();
+ while (resultSet.next()) {
+ Assert.assertTrue(
+ "SHOW TIMESERIES returned a duplicate path",
+ actualTimeseries.add(resultSet.getString(1)));
+ }
+ Assert.assertEquals(
+ Set.of(
+ INVALID_DATABASE + ".valid_device.quality",
+ INVALID_DATABASE + ".valid_device.value"),
+ actualTimeseries);
+ }
+ try (ResultSet resultSet =
+ statement.executeQuery(
+ "select quality, value from " + INVALID_DATABASE +
".valid_device")) {
+ if (miniConversion) {
+ Assert.assertTrue(
+ "Mini conversion must retain the successfully converted
valid prefix",
+ resultSet.next());
+ Assert.assertEquals(1, resultSet.getLong(1));
+ Assert.assertEquals(7, resultSet.getInt(2));
+ Assert.assertNull(
+ "Source mods must also apply to a visible valid prefix",
resultSet.getObject(3));
+ Assert.assertFalse(resultSet.next());
+ } else {
+ Assert.assertFalse(
+ "Normal analysis failure must not execute the LOAD data
phase", resultSet.next());
+ }
+ }
+ }
+ }
+ }
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ deleteDatabase(statement, INVALID_DATABASE);
+ statement.execute("create database " + INVALID_DATABASE);
+ statement.execute("create timeseries " + INVALID_DATABASE +
".valid_after.value INT64");
+ final File validFile = new File(tmpDir, "valid-after-invalid.tsfile");
+ writeTreeFile(validFile, INVALID_DATABASE + ".valid_after", new long[]
{1}, new long[] {42});
+ statement.execute(
+ String.format(
+ "load \"%s\" with
('verify'='false','auto-create-schema'='false','on-success'='none')",
+ validFile.getAbsolutePath()));
+ assertOneTreeValue(statement, INVALID_DATABASE + ".valid_after", 1, 42);
+ }
+ assertLoadTsFileMemoryReturnsTo(loadMemoryBefore);
+ }
+
+ private String buildLoadSql(
+ final File file,
+ final long tabletConversionThreshold,
+ final String databaseLevel,
+ final String onSuccess) {
+ final String thresholdAttribute =
+ tabletConversionThreshold < 0
+ ? ""
+ : ", 'tablet-conversion-threshold'='" + tabletConversionThreshold
+ "'";
+ return String.format(
+ "load \"%s\" with ('database-level'='%s','auto-create-schema'='false',"
+ + "'on-success'='%s'%s)",
+ file.getAbsolutePath(), databaseLevel, onSuccess, thresholdAttribute);
+ }
+
+ private long getMiniConversionCount() {
+ long count = 0;
+ for (final String content :
EnvFactory.getEnv().getMetricPrometheusReporterContents(null)) {
+ if (content == null) {
+ continue;
+ }
+ for (final String line : content.split("\\n")) {
+ final int labelStart = line.indexOf('{');
+ if (labelStart < 0
+ || !line.substring(0, labelStart).endsWith("_count")
+ || !line.contains("load_time_cost")
+ || !line.contains("name=\"analysis_cast_tablets\"")) {
+ continue;
+ }
+ count +=
Math.round(Double.parseDouble(line.substring(line.lastIndexOf(' ') +
1).trim()));
+ }
+ }
+ return count;
+ }
+
+ private long getLoadTsFileUsedMemory() {
+ long usedMemory = 0;
+ boolean foundMetric = false;
+ for (final String content :
EnvFactory.getEnv().getMetricPrometheusReporterContents(null)) {
+ if (content == null) {
+ continue;
+ }
+ for (final String line : content.split("\\n")) {
+ final int labelStart = line.indexOf('{');
+ if (labelStart < 0
+ || !line.substring(0, labelStart).equals("load_mem")
+ || !line.contains("name=\"LoadTsFileUsedMemory\"")) {
+ continue;
+ }
+ foundMetric = true;
+ usedMemory +=
Math.round(Double.parseDouble(line.substring(line.lastIndexOf(' ') + 1)));
+ }
+ }
+ Assert.assertTrue(
+ "LoadTsFileUsedMemory must be exposed by the configured reporter",
foundMetric);
+ return usedMemory;
+ }
+
+ private void assertLoadTsFileMemoryReturnsTo(final long expectedMemory)
+ throws InterruptedException {
+ final long deadline = System.currentTimeMillis() + 10_000L;
+ long actualMemory = getLoadTsFileUsedMemory();
+ while (actualMemory != expectedMemory && System.currentTimeMillis() <
deadline) {
+ Thread.sleep(100L);
+ actualMemory = getLoadTsFileUsedMemory();
+ }
+ Assert.assertEquals(
+ "LOAD memory usage should return to its prior level", expectedMemory,
actualMemory);
+ }
+
+ private void assertThresholdFileContents(final Statement statement, final
String device)
+ throws SQLException {
+ try (ResultSet resultSet =
+ statement.executeQuery("select value from " + device + " order by
time")) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(10000, resultSet.getLong(1));
+ Assert.assertEquals(10, resultSet.getLong(2));
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(20000, resultSet.getLong(1));
+ Assert.assertEquals(20, resultSet.getLong(2));
+ Assert.assertFalse(resultSet.next());
+ }
+ }
+
+ private void assertTreeSeriesValues(
+ final Statement statement,
+ final String device,
+ final int expectedCount,
+ final long startValue)
+ throws SQLException {
+ try (ResultSet resultSet =
+ statement.executeQuery("select value from " + device + " order by
time")) {
+ for (int i = 0; i < expectedCount; i++) {
+ Assert.assertTrue(
+ "A normal-path row was lost during statement reconstruction",
resultSet.next());
+ Assert.assertEquals(100L * (i + 1), resultSet.getLong(1));
+ Assert.assertEquals(startValue + i, resultSet.getLong(2));
+ }
+ Assert.assertFalse(resultSet.next());
+ }
+ }
+
+ private void assertTsFileOrder(final File directory, final String...
expectedFileNames) {
+ final File[] files = directory.listFiles((dir, name) ->
name.endsWith(".tsfile"));
+ Assert.assertNotNull(files);
+ Arrays.sort(files, Comparator.comparing(File::getName));
+ Assert.assertEquals(
+ Arrays.asList(expectedFileNames),
+ Arrays.stream(files).map(File::getName).collect(Collectors.toList()));
+ }
+
+ private void assertOneTreeValue(
+ final Statement statement,
+ final String device,
+ final long expectedTime,
+ final long expectedValue)
+ throws SQLException {
+ try (ResultSet resultSet =
+ statement.executeQuery("select value from " + device + " order by
time")) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(expectedTime, resultSet.getLong(1));
+ Assert.assertEquals(expectedValue, resultSet.getLong(2));
+ Assert.assertFalse(resultSet.next());
+ }
+ }
+
+ private void writeTreeFile(
+ final File file, final String device, final long[] timestamps, final
long[] values)
+ throws Exception {
+ Assert.assertEquals(timestamps.length, values.length);
+ final MeasurementSchema schema = new MeasurementSchema("value",
TSDataType.INT64);
+ try (TsFileWriter writer = new TsFileWriter(file)) {
+ writer.registerTimeseries(new Path(device),
Collections.singletonList(schema));
+ final Tablet tablet = new Tablet(device,
Collections.singletonList(schema));
+ for (int i = 0; i < timestamps.length; i++) {
+ final int row = tablet.getRowSize();
+ tablet.addTimestamp(row, timestamps[i]);
+ tablet.addValue("value", row, values[i]);
+ }
+ writer.writeTree(tablet);
+ }
+ }
+
+ private void writeTypedTreeFile(
+ final File file,
+ final String device,
+ final TSDataType dataType,
+ final long[] timestamps,
+ final Object[] values)
+ throws Exception {
+ Assert.assertEquals(timestamps.length, values.length);
+ final MeasurementSchema schema = new MeasurementSchema("value", dataType);
+ try (TsFileWriter writer = new TsFileWriter(file)) {
+ writer.registerTimeseries(new Path(device),
Collections.singletonList(schema));
+ final Tablet tablet = new Tablet(device,
Collections.singletonList(schema));
+ for (int i = 0; i < timestamps.length; i++) {
+ final int row = tablet.getRowSize();
+ tablet.addTimestamp(row, timestamps[i]);
+ tablet.addValue("value", row, values[i]);
+ }
+ writer.writeTree(tablet);
+ }
+ }
+
+ private void writeTreeDeletion(
+ final File file,
+ final String device,
+ final String measurement,
+ final long startTime,
+ final long endTime)
+ throws IOException, IllegalPathException {
+ try (ModificationFile modifications =
+ new ModificationFile(
+ new File(file.getAbsolutePath() + ModificationFile.FILE_SUFFIX),
false)) {
+ modifications.write(
+ new TreeDeletionEntry(new MeasurementPath(device, measurement),
startTime, endTime));
+ }
+ }
+
+ private void writeInvalidPathTsFile(final File file, final
StringArrayDeviceID invalidDevice)
+ throws Exception {
+ final StringArrayDeviceID validDevice =
+ new StringArrayDeviceID(new String[] {"root", "load_p120",
"valid_device"});
+ try (TsFileIOWriter writer = new TsFileIOWriter(file)) {
+ writer.setSchema(new Schema());
+ writeInvalidPathChunkGroup(writer, validDevice, 1, 7, 42.0);
+ writeInvalidPathChunkGroup(writer, invalidDevice, 2, 1, 1.0);
+ writer.endFile();
+ }
+ }
+
+ private void writeInvalidPathChunkGroup(
+ final TsFileIOWriter writer,
+ final StringArrayDeviceID device,
+ final long timestamp,
+ final int quality,
+ final double value)
+ throws IOException {
+ writer.startChunkGroup(device);
+ final AlignedChunkWriterImpl chunkWriter =
+ new AlignedChunkWriterImpl(
+ List.of(
+ new MeasurementSchema("quality", TSDataType.INT32),
+ new MeasurementSchema("value", TSDataType.DOUBLE)));
+ chunkWriter.getTimeChunkWriter().write(timestamp);
+ chunkWriter.getValueChunkWriterByIndex(0).write(timestamp, quality, false);
+ chunkWriter.getValueChunkWriterByIndex(1).write(timestamp, value, false);
+ chunkWriter.writeToFileWriter(writer);
+ writer.endChunkGroup();
+ }
+
+ private void deleteDatabase(final Statement statement, final String
database) {
+ try {
+ statement.execute("delete database " + database);
+ } catch (SQLException ignored) {
+ // The database may not have been created by the test.
+ }
+ }
+
+ private long[] buildTimes(final int count, final long timeGap) {
+ final long[] times = new long[count];
+ for (int i = 0; i < count; i++) {
+ times[i] = timeGap * (i + 1);
+ }
+ return times;
+ }
+
+ private long[] buildValues(final int count, final long startValue) {
+ final long[] values = new long[count];
+ for (int i = 0; i < count; i++) {
+ values[i] = startValue + i;
+ }
+ return values;
+ }
+
+ private void deleteRecursively(final File file) throws IOException {
+ if (file == null || !file.exists()) {
+ return;
+ }
+ final File[] children = file.listFiles();
+ if (children != null) {
+ for (final File child : children) {
+ deleteRecursively(child);
+ }
+ }
+ Files.deleteIfExists(file.toPath());
+ }
+}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileMiniIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileMiniIT.java
new file mode 100644
index 00000000000..1fdd3ef92bd
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadTsFileMiniIT.java
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.relational.it.db.it;
+
+import
org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBLoadTsFileMiniIT {
+ private static final String TABLE_DATABASE = "load_p116_target";
+ private static final String SESSION_DATABASE = "load_p116_session";
+ private static final String TREE_DATABASE = "root.load_p116_tree";
+
+ private File tmpDir;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+
EnvFactory.getEnv().getConfig().getCommonConfig().setTimePartitionInterval(10_000L);
+
EnvFactory.getEnv().getConfig().getCommonConfig().setEnforceStrongPassword(false);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setAutoCreateSchemaEnabled(true)
+ .setDatanodeMemoryProportion("3:3:1:1:1:0");
+ EnvFactory.getEnv()
+ .getConfig()
+ .getDataNodeConfig()
+ .setMetricReporterType(Collections.singletonList("PROMETHEUS"))
+ .setMetricPrometheusReporterAsyncUpdate(false);
+ EnvFactory.getEnv()
+ .getConfig()
+ .getConfigNodeConfig()
+ .setMetricReporterType(Collections.singletonList("PROMETHEUS"));
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ tmpDir = Files.createTempDirectory("load-mini").toFile();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("drop database if exists " + TABLE_DATABASE);
+ statement.execute("drop database if exists " + SESSION_DATABASE);
+ try (Connection treeConnection = EnvFactory.getEnv().getConnection();
+ Statement treeStatement = treeConnection.createStatement()) {
+ try {
+ treeStatement.execute("delete database " + TREE_DATABASE);
+ } catch (SQLException ignored) {
+ // The table-only case does not create the Tree database.
+ }
+ }
+ } finally {
+ deleteRecursively(tmpDir);
+ }
+ }
+
+ @AfterClass
+ public static void tearDownClass() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testTableMiniConversionHonorsExplicitAndSessionDatabases()
throws Exception {
+ final File explicitFile = new File(tmpDir, "mini-explicit.tsfile");
+ final File sessionFile = new File(tmpDir, "mini-session.tsfile");
+ writeMiniTableFile(explicitFile, "mini_explicit", true);
+ writeMiniTableFile(sessionFile, "mini_session", false);
+ final long threshold = Math.max(explicitFile.length(),
sessionFile.length());
+
+ try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database " + TABLE_DATABASE);
+ statement.execute("create database " + SESSION_DATABASE);
+ statement.execute("use " + TABLE_DATABASE);
+ statement.execute(
+ "create table mini_explicit (site STRING TAG, zone STRING TAG,
reading INT32 FIELD)");
+ statement.execute("use " + SESSION_DATABASE);
+ statement.execute(
+ "create table mini_session (site STRING TAG, zone STRING TAG,
reading INT32 FIELD)");
+ }
+
+ final long conversionsBefore = getMiniConversionCount();
+ try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("use " + SESSION_DATABASE);
+ statement.execute(
+ String.format(
+ "load '%s' with ('database'='%s','database-level'='2',"
+ +
"'tablet-conversion-threshold'='%d','auto-create-schema'='false',"
+ + "'on-success'='none')",
+ explicitFile.getAbsolutePath(), TABLE_DATABASE, threshold));
+ try (ResultSet resultSet =
+ statement.executeQuery("select time, site, zone, reading from
mini_session")) {
+ Assert.assertFalse("Explicit database LOAD changed the session
database", resultSet.next());
+ }
+
+ statement.execute(
+ String.format(
+ "load '%s' with
('database-level'='2','tablet-conversion-threshold'='%d',"
+ + "'auto-create-schema'='false','on-success'='none')",
+ sessionFile.getAbsolutePath(), threshold));
+ try (ResultSet resultSet =
+ statement.executeQuery("select time, site, zone, reading from
mini_session")) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(20, resultSet.getLong("time"));
+ Assert.assertEquals("east", resultSet.getString("site"));
+ Assert.assertNull(resultSet.getString("zone"));
+ Assert.assertEquals(9, resultSet.getInt("reading"));
+ Assert.assertFalse(resultSet.next());
+ }
+ try (ResultSet resultSet = statement.executeQuery("show tables")) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals("mini_session", resultSet.getString(1));
+ Assert.assertFalse(resultSet.next());
+ }
+ }
+
+ try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("use " + TABLE_DATABASE);
+ try (ResultSet resultSet =
+ statement.executeQuery("select time, site, zone, reading from
mini_explicit")) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(11, resultSet.getLong("time"));
+ Assert.assertEquals("south", resultSet.getString("site"));
+ Assert.assertEquals("south-1", resultSet.getString("zone"));
+ Assert.assertEquals(8, resultSet.getInt("reading"));
+ Assert.assertFalse(resultSet.next());
+ }
+ }
+ Assert.assertEquals(2, getMiniConversionCount() - conversionsBefore);
+ Assert.assertTrue(explicitFile.exists());
+ Assert.assertTrue(sessionFile.exists());
+ }
+
+ @Test
+ public void testMixedMiniAndNormalFilesKeepModelState() throws Exception {
+ final File loadDir = new File(tmpDir, "mixed-mini");
+ Assert.assertTrue(loadDir.mkdir());
+ final File firstMini = new File(loadDir, "01-tree-mini.tsfile");
+ final File normalTable = new File(loadDir, "02-table-normal.tsfile");
+ final File lastMini = new File(loadDir, "03-tree-mini.tsfile");
+ writeTreeFile(firstMini, TREE_DATABASE + ".before", 1, 101);
+ writeMiniTableFile(normalTable, "normal_table", true);
+ writeTreeFile(lastMini, TREE_DATABASE + ".after", 3, 303);
+ assertTsFileOrder(
+ loadDir, "01-tree-mini.tsfile", "02-table-normal.tsfile",
"03-tree-mini.tsfile");
+ final long threshold = Math.max(firstMini.length(), lastMini.length());
+ Assert.assertTrue(normalTable.length() > threshold);
+ createTreeSchema("before", "after");
+ final long conversionsBefore = getMiniConversionCount();
+ try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database " + TABLE_DATABASE);
+ statement.execute("create database " + SESSION_DATABASE);
+ statement.execute("use " + TABLE_DATABASE);
+ statement.execute(
+ "create table normal_table (site STRING TAG, zone STRING TAG,
reading INT32 FIELD)");
+ statement.execute("use " + SESSION_DATABASE);
+ statement.execute(
+ String.format(
+ "load '%s' with ('database'='%s','database-level'='2',"
+ +
"'tablet-conversion-threshold'='%d','auto-create-schema'='false',"
+ + "'convert-on-type-mismatch'='false','on-success'='none')",
+ loadDir.getAbsolutePath(), TABLE_DATABASE, threshold));
+ statement.execute("create table session_probe (site STRING TAG, reading
INT32 FIELD)");
+ statement.execute("insert into session_probe(time, site, reading) values
(20, 'session', 2)");
+ try (ResultSet resultSet = statement.executeQuery("select reading from
session_probe")) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(2, resultSet.getInt(1));
+ Assert.assertFalse(resultSet.next());
+ }
+ statement.execute("use " + TABLE_DATABASE);
+ try (ResultSet resultSet =
+ statement.executeQuery("select time, site, zone, reading from
normal_table")) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(11, resultSet.getLong("time"));
+ Assert.assertEquals("south", resultSet.getString("site"));
+ Assert.assertEquals("south-1", resultSet.getString("zone"));
+ Assert.assertEquals(8, resultSet.getInt("reading"));
+ Assert.assertFalse(resultSet.next());
+ }
+ }
+ Assert.assertEquals(2, getMiniConversionCount() - conversionsBefore);
+ assertTreeValue(TREE_DATABASE + ".before", 1, 101);
+ assertTreeValue(TREE_DATABASE + ".after", 3, 303);
+ Assert.assertTrue(firstMini.exists());
+ Assert.assertTrue(normalTable.exists());
+ Assert.assertTrue(lastMini.exists());
+ }
+
+ private void writeMiniTableFile(
+ final File file, final String tableName, final boolean withDeletion)
throws Exception {
+ final List<IMeasurementSchema> schemas =
+ Arrays.asList(
+ new MeasurementSchema("site", TSDataType.STRING),
+ new MeasurementSchema("zone", TSDataType.STRING),
+ new MeasurementSchema("reading", TSDataType.INT32));
+ final List<ColumnCategory> categories =
+ Arrays.asList(ColumnCategory.TAG, ColumnCategory.TAG,
ColumnCategory.FIELD);
+ try (TsFileWriter writer = new TsFileWriter(file)) {
+ writeTable(
+ writer,
+ tableName,
+ schemas,
+ categories,
+ withDeletion
+ ? Arrays.asList(
+ new Object[] {10L, "north", null, 7}, new Object[] {11L,
"south", "south-1", 8})
+ : Collections.singletonList(new Object[] {20L, "east", null,
9}));
+ }
+ if (withDeletion) {
+ try (ModificationFile modifications =
+ new ModificationFile(
+ new File(file.getAbsolutePath() + ModificationFile.FILE_SUFFIX),
false)) {
+ modifications.write(
+ new TableDeletionEntry(new DeletionPredicate(tableName), new
TimeRange(10, 10)));
+ }
+ }
+ }
+
+ private long getMiniConversionCount() {
+ long count = 0;
+ for (final String content :
EnvFactory.getEnv().getMetricPrometheusReporterContents(null)) {
+ if (content == null) {
+ continue;
+ }
+ for (final String line : content.split("\\n")) {
+ final int labelStart = line.indexOf('{');
+ if (labelStart < 0
+ || !line.substring(0, labelStart).endsWith("_count")
+ || !line.contains("load_time_cost")
+ || !line.contains("analysis_cast_tablets")) {
+ continue;
+ }
+ count +=
Math.round(Double.parseDouble(line.substring(line.lastIndexOf(' ') +
1).trim()));
+ }
+ }
+ return count;
+ }
+
+ private void createTreeSchema(final String... devices) throws SQLException {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database " + TREE_DATABASE);
+ for (final String device : devices) {
+ statement.execute("create timeseries " + TREE_DATABASE + "." + device
+ ".value INT64");
+ }
+ }
+ }
+
+ private void writeTable(
+ final TsFileWriter writer,
+ final String tableName,
+ final List<IMeasurementSchema> schemas,
+ final List<ColumnCategory> categories,
+ final List<Object[]> rows)
+ throws IOException, WriteProcessException {
+ writer.registerTableSchema(new TableSchema(tableName, schemas,
categories));
+ final List<String> names =
+
schemas.stream().map(IMeasurementSchema::getMeasurementName).collect(Collectors.toList());
+ final List<TSDataType> types =
+
schemas.stream().map(IMeasurementSchema::getType).collect(Collectors.toList());
+ final Tablet tablet = new Tablet(tableName, names, types, categories);
+ for (final Object[] values : rows) {
+ final int row = tablet.getRowSize();
+ tablet.addTimestamp(row, (Long) values[0]);
+ for (int column = 0; column < schemas.size(); column++) {
+ tablet.addValue(names.get(column), row, values[column + 1]);
+ }
+ }
+ writer.writeTable(tablet);
+ }
+
+ private void assertTsFileOrder(final File directory, final String...
expectedNames) {
+ final File[] files = directory.listFiles((dir, name) ->
name.endsWith(".tsfile"));
+ Assert.assertNotNull(files);
+ Arrays.sort(files, Comparator.comparing(File::getName));
+ Assert.assertEquals(
+ Arrays.asList(expectedNames),
+ Arrays.stream(files).map(File::getName).collect(Collectors.toList()));
+ }
+
+ private void writeTreeFile(
+ final File file, final String device, final long timestamp, final long
value)
+ throws Exception {
+ final MeasurementSchema schema = new MeasurementSchema("value",
TSDataType.INT64);
+ try (TsFileWriter writer = new TsFileWriter(file)) {
+ writer.registerTimeseries(new Path(device),
Collections.singletonList(schema));
+ final Tablet tablet = new Tablet(device,
Collections.singletonList(schema));
+ tablet.addTimestamp(0, timestamp);
+ tablet.addValue(0, 0, value);
+ writer.writeTree(tablet);
+ }
+ }
+
+ private void assertTreeValue(
+ final String device, final long expectedTime, final long expectedValue)
throws SQLException {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement();
+ ResultSet resultSet =
+ statement.executeQuery("select value from " + device + " order by
time")) {
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(expectedTime, resultSet.getLong(1));
+ Assert.assertEquals(expectedValue, resultSet.getLong(2));
+ Assert.assertFalse(resultSet.next());
+ }
+ }
+
+ private void deleteRecursively(final File file) throws IOException {
+ if (file == null || !file.exists()) {
+ return;
+ }
+ final File[] children = file.listFiles();
+ if (children != null) {
+ for (final File child : children) {
+ deleteRecursively(child);
+ }
+ }
+ Files.deleteIfExists(file.toPath());
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
index 551130ad6b9..d7c15da31f6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
@@ -478,9 +478,24 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
this.isTableModelTsFileReliableIndex = i;
if (0 <= tabletConversionThresholdBytes
- && tsFile.length() <= tabletConversionThresholdBytes
- && handleSingleMiniFile(i)) {
- return;
+ && tsFile.length() <= tabletConversionThresholdBytes) {
+ if (!isTableModelFile
+ && !isConvertOnTypeMismatch
+ && treeSchemaAutoCreatorAndVerifier != null) {
+ treeSchemaAutoCreatorAndVerifier.flush();
+ }
+ final long startTime = System.nanoTime();
+ try {
+ if (!isTableModelFile) {
+ checkMiniFileTreeWritePermission(tsFile, reader);
+ }
+ if (handleSingleMiniFile(i)) {
+ return;
+ }
+ } finally {
+ LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost(
+ LoadTsFileCostMetricsSet.ANALYSIS_CAST_TABLETS,
System.nanoTime() - startTime);
+ }
}
if (isTableModelFile) {
@@ -536,40 +551,67 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
}
private boolean handleSingleMiniFile(final int i) throws
FileNotFoundException {
- final long startTime = System.nanoTime();
- try {
- final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
- new LoadTsFileDataTypeConverter(context, isGeneratedByPipe);
-
- final TSStatus status =
- isTableModelTsFile.get(i)
- ? loadTsFileDataTypeConverter
-
.convertForTableModel(createTableConversionStatement(tsFiles.get(i)))
- .orElse(null)
- : loadTsFileDataTypeConverter
-
.convertForTreeModel(createTreeConversionStatement(tsFiles.get(i)))
- .orElse(null);
-
- if (status == null || !loadTsFileDataTypeConverter.isSuccessful(status))
{
- LOGGER.warn(
- DataNodeQueryMessages
-
.LOAD_FAILED_TO_CONVERT_MINI_TSFILE_ARG_TO_TABLETS_FROM_STATEMENT_ARG_STATUS_ARG,
- tsFiles.get(i).getPath(),
- isTableModelStatement ? loadTsFileTableStatement :
loadTsFileTreeStatement,
- status);
- return false;
- }
+ final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter =
+ new LoadTsFileDataTypeConverter(context, isGeneratedByPipe);
- // A mark of successful conversion
- isMiniTsFile.set(i, Boolean.TRUE);
- isMiniTsFileConverted = true;
+ final TSStatus status =
+ isTableModelTsFile.get(i)
+ ? loadTsFileDataTypeConverter
+
.convertForTableModel(createTableConversionStatement(tsFiles.get(i)))
+ .orElse(null)
+ : loadTsFileDataTypeConverter
+
.convertForTreeModel(createTreeConversionStatement(tsFiles.get(i)))
+ .orElse(null);
- addTsFileResource(null);
- addWritePointCount(0);
- return true;
- } finally {
- LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost(
- LoadTsFileCostMetricsSet.ANALYSIS_CAST_TABLETS, System.nanoTime() -
startTime);
+ if (status == null || !loadTsFileDataTypeConverter.isSuccessful(status)) {
+ LOGGER.warn(
+ DataNodeQueryMessages
+
.LOAD_FAILED_TO_CONVERT_MINI_TSFILE_ARG_TO_TABLETS_FROM_STATEMENT_ARG_STATUS_ARG,
+ tsFiles.get(i).getPath(),
+ isTableModelStatement ? loadTsFileTableStatement :
loadTsFileTreeStatement,
+ status);
+ return false;
+ }
+
+ // A mark of successful conversion
+ isMiniTsFile.set(i, Boolean.TRUE);
+ isMiniTsFileConverted = true;
+
+ addTsFileResource(null);
+ addWritePointCount(0);
+ return true;
+ }
+
+ private void checkMiniFileTreeWritePermission(
+ final File tsFile, final TsFileSequenceReader reader)
+ throws IOException, LoadAnalyzeException, AuthException {
+ final TsFileResource tsFileResource = constructTsFileResource(reader,
tsFile);
+ final TreeSchemaAutoCreatorAndVerifier verifier =
getOrCreateTreeSchemaVerifier();
+ verifier.setCurrentModificationsAndTimeIndex(tsFileResource, reader);
+
+ final TsFileSequenceReaderTimeseriesMetadataIterator
timeseriesMetadataIterator =
+ new TsFileSequenceReaderTimeseriesMetadataIterator(
+ reader,
+ true,
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getLoadTsFileAnalyzeSchemaBatchReadTimeSeriesMetadataCount());
+ while (timeseriesMetadataIterator.hasNext()) {
+ final Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadata
=
+ timeseriesMetadataIterator.next();
+ TsFileResourceUtils.updateTsFileResource(
+ device2TimeseriesMetadata,
+ tsFileResource,
+
IoTDBDescriptor.getInstance().getConfig().isCacheLastValuesForLoad());
+ verifier.setCurrentTimeIndex(tsFileResource.getTimeIndex());
+ for (final Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry :
+ device2TimeseriesMetadata.entrySet()) {
+ try {
+
verifier.checkWritePermission(Collections.singletonMap(entry.getKey(),
entry.getValue()));
+ } catch (LoadAnalyzeInvalidPathException ignored) {
+ // Let conversion report invalid paths in file order, after
processing earlier tablets.
+ }
+ }
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
index 3b16720b4a4..220c17f9db8 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java
@@ -335,7 +335,7 @@ public class LoadTsFileDataTypeConverter {
new UserEntity(session.getUserId(), session.getUsername(),
session.getClientAddress()),
ZoneId.systemDefault(),
session.getClientVersion(),
- session.getDatabaseName(),
+ null,
SqlDialect.TREE,
allowAutoCreateSchema
&& session.isEnableAutoCreateSchema()