This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 876b22b10c4 Fix LOAD rate limiting to account for source TsFile bytes
(#19141) (#19147)
876b22b10c4 is described below
commit 876b22b10c430b43bb6e15ae1ab1168a8f8f5150
Author: Caideyipi <[email protected]>
AuthorDate: Fri Nov 6 10:56:34 2026 +0800
Fix LOAD rate limiting to account for source TsFile bytes (#19141) (#19147)
---
.../db/storageengine/dataregion/DataRegion.java | 2 +-
.../storageengine/dataregion/DataRegionTest.java | 55 ++++++++++++++++++++++
2 files changed, 56 insertions(+), 1 deletion(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index c000b246c92..986d06fa817 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -3529,7 +3529,7 @@ public class DataRegion implements IDataRegionForQuery {
targetFile.getAbsolutePath());
}
-
LoadTsFileRateLimiter.getInstance().acquire(tsFileResource.getTsFile().length());
+ LoadTsFileRateLimiter.getInstance().acquire(tsFileToLoad.length());
// move file from sync dir to data dir
if (!targetFile.getParentFile().exists()) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
index b54edbca14d..93223cc8ccf 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
@@ -65,10 +65,12 @@ import
org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
+import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet;
import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager;
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.db.utils.constant.TestConstant;
+import org.apache.iotdb.metrics.type.Counter;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -85,7 +87,9 @@ import org.apache.tsfile.read.expression.QueryExpression;
import org.apache.tsfile.read.query.dataset.QueryDataSet;
import org.apache.tsfile.read.reader.IPointReader;
import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.TsFileWriter;
import org.apache.tsfile.write.record.TSRecord;
+import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.record.datapoint.DataPoint;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
@@ -101,6 +105,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -189,6 +194,56 @@ public class DataRegionTest {
return insertRowNode;
}
+ @Test
+ public void testLoadRateLimiterAccountsForSourceFileBeforeTransfer() throws
Exception {
+ final LoadTsFileCostMetricsSet metrics =
LoadTsFileCostMetricsSet.getInstance();
+ final Field counterField =
LoadTsFileCostMetricsSet.class.getDeclaredField("diskIOCounter");
+ counterField.setAccessible(true);
+ final Counter originalCounter = (Counter) counterField.get(metrics);
+ final double originalThroughput =
config.getLoadWriteThroughputBytesPerSecond();
+ try {
+ config.setLoadWriteThroughputBytesPerSecond(0);
+ for (boolean deleteOrigin : new boolean[] {false, true}) {
+ final File sourceDir =
+ new File(TestConstant.OUTPUT_DATA_DIR, "load-rate-source-" +
deleteOrigin);
+ Files.createDirectories(sourceDir.toPath());
+ final File source = new File(sourceDir, "1-1-0-0.tsfile");
+ final MeasurementSchema schema = new MeasurementSchema(measurementId,
TSDataType.INT32);
+ try (TsFileWriter writer = new TsFileWriter(source)) {
+ writer.registerAlignedTimeseries(new Path(deviceId),
Collections.singletonList(schema));
+ final Tablet tablet = new Tablet(deviceId,
Collections.singletonList(schema), 1);
+ tablet.addTimestamp(0, 1);
+ tablet.addValue(measurementId, 0, 42);
+ tablet.rowSize = 1;
+ writer.writeAligned(tablet);
+ }
+ final TsFileResource resource = new TsFileResource(source);
+ resource.setStatus(TsFileResourceStatus.NORMAL);
+ resource.updateStartTime(new PlainDeviceID(deviceId), 1);
+ resource.updateEndTime(new PlainDeviceID(deviceId), 1);
+ resource.serialize();
+ final byte[] sourceBytes = Files.readAllBytes(source.toPath());
+ Assert.assertTrue(sourceBytes.length > 0);
+ // acquire records the same byte count in this counter, without a
timing-based assertion.
+ final Counter counter = Mockito.mock(Counter.class);
+ counterField.set(metrics, counter);
+
+ dataRegion.loadNewTsFile(resource, deleteOrigin, false, true);
+
+ Mockito.verify(counter).inc(sourceBytes.length);
+ Mockito.verifyNoMoreInteractions(counter);
+ Assert.assertNotEquals(source.getAbsolutePath(),
resource.getTsFile().getAbsolutePath());
+ Assert.assertArrayEquals(sourceBytes,
Files.readAllBytes(resource.getTsFile().toPath()));
+ Assert.assertEquals(!deleteOrigin, source.exists());
+ Assert.assertEquals(
+ !deleteOrigin, new File(source +
TsFileResource.RESOURCE_SUFFIX).exists());
+ }
+ } finally {
+ counterField.set(metrics, originalCounter);
+ config.setLoadWriteThroughputBytesPerSecond(originalThroughput);
+ }
+ }
+
@Test
public void testRejectedRowFlushesOnlySelectedOtherMemtable() throws
Exception {
assertRejectedInsertionFlushesOnlySelectedOtherMemtable(false);