This is an automated email from the ASF dual-hosted git repository.
justinchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 67c450be53f Load: Add IT test about Load file across multiple time
partitions (#16253)
67c450be53f is described below
commit 67c450be53f83201c98569f4e75e45d5fa3072aa
Author: Zhenyu Luo <[email protected]>
AuthorDate: Tue Aug 26 11:39:48 2025 +0800
Load: Add IT test about Load file across multiple time partitions (#16253)
* add IT
* add IT
---
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 83 ++++++++++++++++++++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 8 +--
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 +-
3 files changed, 89 insertions(+), 6 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
index a89becf1aa5..7d5f7be8a1e 100644
---
a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
@@ -224,6 +224,9 @@ public class IoTDBLoadTsFileIT {
generator.generateData(SchemaConfig.DEVICE_2, 10000, PARTITION_INTERVAL
/ 10_000, false);
generator.generateData(SchemaConfig.DEVICE_3, 10000, PARTITION_INTERVAL
/ 10_000, false);
generator.generateData(SchemaConfig.DEVICE_4, 10000, PARTITION_INTERVAL
/ 10_000, true);
+ for (int i = 0; i < 10000; i++) {
+ generator.generateData(SchemaConfig.DEVICE_4, 1, PARTITION_INTERVAL -
10, true);
+ }
writtenPoint2 = generator.getTotalNumber();
}
@@ -255,6 +258,86 @@ public class IoTDBLoadTsFileIT {
}
}
+ @Test
+ public void testLoadAcrossMultipleTimePartitions() throws Exception {
+ registerSchema();
+
+ final long writtenPoint1;
+ // device 0, device 1, sg 0
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
+ generator.registerTimeseries(
+ SchemaConfig.DEVICE_0,
+ Arrays.asList(
+ SchemaConfig.MEASUREMENT_00,
+ SchemaConfig.MEASUREMENT_01,
+ SchemaConfig.MEASUREMENT_02,
+ SchemaConfig.MEASUREMENT_03,
+ SchemaConfig.MEASUREMENT_04,
+ SchemaConfig.MEASUREMENT_05,
+ SchemaConfig.MEASUREMENT_06,
+ SchemaConfig.MEASUREMENT_07));
+ generator.registerAlignedTimeseries(
+ SchemaConfig.DEVICE_1,
+ Arrays.asList(
+ SchemaConfig.MEASUREMENT_10,
+ SchemaConfig.MEASUREMENT_11,
+ SchemaConfig.MEASUREMENT_12,
+ SchemaConfig.MEASUREMENT_13,
+ SchemaConfig.MEASUREMENT_14,
+ SchemaConfig.MEASUREMENT_15,
+ SchemaConfig.MEASUREMENT_16,
+ SchemaConfig.MEASUREMENT_17));
+ for (int i = 0; i < 1000; i++) {
+ generator.generateData(SchemaConfig.DEVICE_0, 1, PARTITION_INTERVAL -
10, true);
+ }
+ for (int i = 0; i < 1000; i++) {
+ generator.generateData(SchemaConfig.DEVICE_1, 1, PARTITION_INTERVAL -
10, true);
+ }
+ writtenPoint1 = generator.getTotalNumber();
+ }
+
+ final long writtenPoint2;
+ // device 2, device 3, device4, sg 1
+ try (final TsFileGenerator generator =
+ new TsFileGenerator(new File(tmpDir, "2-0-0-0.tsfile"))) {
+ generator.registerTimeseries(
+ SchemaConfig.DEVICE_2,
Collections.singletonList(SchemaConfig.MEASUREMENT_20));
+ generator.registerTimeseries(
+ SchemaConfig.DEVICE_3,
Collections.singletonList(SchemaConfig.MEASUREMENT_30));
+ generator.registerAlignedTimeseries(
+ SchemaConfig.DEVICE_4,
Collections.singletonList(SchemaConfig.MEASUREMENT_40));
+ for (int i = 0; i < 1000; i++) {
+ generator.generateData(SchemaConfig.DEVICE_2, 1, PARTITION_INTERVAL -
10, true);
+ }
+ for (int i = 0; i < 1000; i++) {
+ generator.generateData(SchemaConfig.DEVICE_3, 1, PARTITION_INTERVAL -
10, true);
+ }
+ for (int i = 0; i < 1000; i++) {
+ generator.generateData(SchemaConfig.DEVICE_4, 1, PARTITION_INTERVAL -
10, true);
+ }
+ writtenPoint2 = generator.getTotalNumber();
+ }
+
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+
+ statement.execute(String.format("load \"%s\" sglevel=2",
tmpDir.getAbsolutePath()));
+
+ try (final ResultSet resultSet =
+ statement.executeQuery("select count(*) from root.sg.** group by
level=1,2")) {
+ if (resultSet.next()) {
+ long sg1Count = resultSet.getLong("count(root.sg.test_0.*.*)");
+ Assert.assertEquals(writtenPoint1, sg1Count);
+ long sg2Count = resultSet.getLong("count(root.sg.test_1.*.*)");
+ Assert.assertEquals(writtenPoint2, sg2Count);
+ } else {
+ Assert.fail("This ResultSet is empty.");
+ }
+ }
+ }
+ }
+
@Test
public void testLoadWithExtendTemplate() throws Exception {
final long writtenPoint1;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 0d9ed8ca883..fe97abc0251 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -1112,7 +1112,7 @@ public class IoTDBConfig {
private long loadTableSchemaCacheSizeInBytes = 2 * 1024 * 1024L; // 2MB
- private long loadMeasurementCacheSizeInBytes = 2 * 1024 * 1024L; // 2MB
+ private long loadMeasurementIdCacheSizeInBytes = 2 * 1024 * 1024L; // 2MB
private String[] loadActiveListeningDirs =
new String[] {
@@ -4008,11 +4008,11 @@ public class IoTDBConfig {
}
public long getLoadMeasurementIdCacheSizeInBytes() {
- return loadMeasurementCacheSizeInBytes;
+ return loadMeasurementIdCacheSizeInBytes;
}
- public void setLoadMeasurementCacheSizeInBytes(long
loadMeasurementCacheSizeInBytes) {
- this.loadMeasurementCacheSizeInBytes = loadMeasurementCacheSizeInBytes;
+ public void setLoadMeasurementIdCacheSizeInBytes(long
loadMeasurementIdCacheSizeInBytes) {
+ this.loadMeasurementIdCacheSizeInBytes = loadMeasurementIdCacheSizeInBytes;
}
public void setPipeReceiverFileDirs(String[] pipeReceiverFileDirs) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 5dba46abf99..738c8f58ec6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -2313,10 +2313,10 @@ public class IoTDBDescriptor {
"load_table_schema_cache_size_in_bytes",
Long.toString(conf.getLoadTableSchemaCacheSizeInBytes()))));
- conf.setLoadMeasurementCacheSizeInBytes(
+ conf.setLoadMeasurementIdCacheSizeInBytes(
Long.parseLong(
properties.getProperty(
- "load_measurement_cache_size_in_bytes",
+ "load_measurement_id_cache_size_in_bytes",
Long.toString(conf.getLoadMeasurementIdCacheSizeInBytes()))));
conf.setLoadActiveListeningMaxThreadNum(
