This is an automated email from the ASF dual-hosted git repository.

Caideyipi pushed a commit to branch 
codex/backport-load-rate-limiting-19141-dev13
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 1410b75918cc7eb03699a16676b5437002279c4c
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 5 16:47:04 2026 +0800

    Fix LOAD rate limiting to account for source TsFile bytes (#19141)
    
    (cherry picked from commit c4086c5df64dcebf052b3f08ed462ed5293ab1cb)
---
 .../db/storageengine/dataregion/DataRegion.java    |  2 +-
 .../storageengine/dataregion/DataRegionTest.java   | 55 ++++++++++++++++++++++
 2 files changed, 56 insertions(+), 1 deletion(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index c000b246c92..986d06fa817 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -3529,7 +3529,7 @@ public class DataRegion implements IDataRegionForQuery {
           targetFile.getAbsolutePath());
     }
 
-    
LoadTsFileRateLimiter.getInstance().acquire(tsFileResource.getTsFile().length());
+    LoadTsFileRateLimiter.getInstance().acquire(tsFileToLoad.length());
 
     // move file from sync dir to data dir
     if (!targetFile.getParentFile().exists()) {
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
index b54edbca14d..112a44879ac 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
@@ -65,10 +65,12 @@ import 
org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
+import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet;
 import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager;
 import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
 import org.apache.iotdb.db.utils.EnvironmentUtils;
 import org.apache.iotdb.db.utils.constant.TestConstant;
+import org.apache.iotdb.metrics.type.Counter;
 import org.apache.iotdb.rpc.RpcUtils;
 import org.apache.iotdb.rpc.TSStatusCode;
 
@@ -85,7 +87,9 @@ import org.apache.tsfile.read.expression.QueryExpression;
 import org.apache.tsfile.read.query.dataset.QueryDataSet;
 import org.apache.tsfile.read.reader.IPointReader;
 import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.TsFileWriter;
 import org.apache.tsfile.write.record.TSRecord;
+import org.apache.tsfile.write.record.Tablet;
 import org.apache.tsfile.write.record.datapoint.DataPoint;
 import org.apache.tsfile.write.schema.MeasurementSchema;
 import org.junit.After;
@@ -101,6 +105,7 @@ import org.slf4j.LoggerFactory;
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Field;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -189,6 +194,56 @@ public class DataRegionTest {
     return insertRowNode;
   }
 
+  @Test
+  public void testLoadRateLimiterAccountsForSourceFileBeforeTransfer() throws 
Exception {
+    final LoadTsFileCostMetricsSet metrics = 
LoadTsFileCostMetricsSet.getInstance();
+    final Field counterField = 
LoadTsFileCostMetricsSet.class.getDeclaredField("diskIOCounter");
+    counterField.setAccessible(true);
+    final Counter originalCounter = (Counter) counterField.get(metrics);
+    final double originalThroughput = 
config.getLoadWriteThroughputBytesPerSecond();
+    try {
+      config.setLoadWriteThroughputBytesPerSecond(0);
+      for (boolean deleteOrigin : new boolean[] {false, true}) {
+        final File sourceDir =
+            new File(TestConstant.OUTPUT_DATA_DIR, "load-rate-source-" + 
deleteOrigin);
+        Files.createDirectories(sourceDir.toPath());
+        final File source = new File(sourceDir, "1-1-0-0.tsfile");
+        final MeasurementSchema schema = new MeasurementSchema(measurementId, 
TSDataType.INT32);
+        try (TsFileWriter writer = new TsFileWriter(source)) {
+          writer.registerAlignedTimeseries(new Path(deviceId), 
Collections.singletonList(schema));
+          final Tablet tablet = new Tablet(deviceId, 
Collections.singletonList(schema), 1);
+          tablet.addTimestamp(0, 1);
+          tablet.addValue(measurementId, 0, 42);
+          writer.writeAligned(tablet);
+        }
+        final TsFileResource resource = new TsFileResource(source);
+        resource.setStatus(TsFileResourceStatus.NORMAL);
+        final PlainDeviceID device = new PlainDeviceID(deviceId);
+        resource.updateStartTime(device, 1);
+        resource.updateEndTime(device, 1);
+        resource.serialize();
+        final byte[] sourceBytes = Files.readAllBytes(source.toPath());
+        Assert.assertTrue(sourceBytes.length > 0);
+        // acquire records the same byte count in this counter, without a 
timing-based assertion.
+        final Counter counter = Mockito.mock(Counter.class);
+        counterField.set(metrics, counter);
+
+        dataRegion.loadNewTsFile(resource, deleteOrigin, false, true);
+
+        Mockito.verify(counter).inc(sourceBytes.length);
+        Mockito.verifyNoMoreInteractions(counter);
+        Assert.assertNotEquals(source.getAbsolutePath(), 
resource.getTsFile().getAbsolutePath());
+        Assert.assertArrayEquals(sourceBytes, 
Files.readAllBytes(resource.getTsFile().toPath()));
+        Assert.assertEquals(!deleteOrigin, source.exists());
+        Assert.assertEquals(
+            !deleteOrigin, new File(source + 
TsFileResource.RESOURCE_SUFFIX).exists());
+      }
+    } finally {
+      counterField.set(metrics, originalCounter);
+      config.setLoadWriteThroughputBytesPerSecond(originalThroughput);
+    }
+  }
+
   @Test
   public void testRejectedRowFlushesOnlySelectedOtherMemtable() throws 
Exception {
     assertRejectedInsertionFlushesOnlySelectedOtherMemtable(false);

Reply via email to