This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch rc/1.3.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rc/1.3.3 by this push:
new a61fc126bdc fix compaction file metrics (#13692)
a61fc126bdc is described below
commit a61fc126bdccb40bf3c7f07994a7c0965c509009
Author: shuwenwei <[email protected]>
AuthorDate: Thu Oct 10 10:25:10 2024 +0800
fix compaction file metrics (#13692)
---
.../task/InsertionCrossSpaceCompactionTask.java | 35 ++++++++-------
.../compaction/execute/utils/CompactionUtils.java | 18 +++++---
.../cross/InsertionCrossSpaceCompactionTest.java | 52 ++++++++++++++++++++++
.../conf/iotdb-system.properties.template | 2 +-
4 files changed, 83 insertions(+), 24 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
index 1f7e2b9c018..634ff2a4f8b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
@@ -165,16 +165,7 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
Collections.singletonList(unseqFileToInsert),
Collections.singletonList(targetFile));
lockWrite(Collections.singletonList(unseqFileToInsert));
- CompactionUtils.deleteSourceTsFileAndUpdateFileMetrics(
- Collections.singletonList(unseqFileToInsert), false);
-
- FileMetrics.getInstance()
- .addTsFile(
- targetFile.getDatabaseName(),
- targetFile.getDataRegionId(),
- targetFile.getTsFileSize(),
- true,
- targetFile.getTsFile().getName());
+ CompactionUtils.deleteTsFileResourceWithoutLock(unseqFileToInsert);
double costTime = (System.currentTimeMillis() - startTime) / 1000.0d;
LOGGER.info(
@@ -199,6 +190,9 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
} catch (IOException e) {
handleException(LOGGER, e);
}
+ if (targetFile != null && targetFile.tsFileExists()) {
+ updateFileMetrics();
+ }
targetFile.setStatus(TsFileResourceStatus.NORMAL);
}
return isSuccess;
@@ -314,9 +308,6 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
if (targetFile == null) {
return;
}
- if (targetFile.tsFileExists()) {
- FileMetrics.getInstance().deleteTsFile(true,
Collections.singletonList(targetFile));
- }
// delete target file
if (!deleteTsFileOnDisk(targetFile)) {
throw new CompactionRecoverException(
@@ -328,9 +319,6 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
if (unseqFileToInsert == null) {
return;
}
- if (recoverMemoryStatus && unseqFileToInsert.tsFileExists()) {
- FileMetrics.getInstance().deleteTsFile(false,
Collections.singletonList(unseqFileToInsert));
- }
if (!deleteTsFileOnDisk(unseqFileToInsert)) {
throw new CompactionRecoverException("source files cannot be deleted
successfully");
}
@@ -381,4 +369,19 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
holdWriteLockList.add(tsFileResource);
}
}
+
+ private void updateFileMetrics() {
+ // Here the target file is used for updating metrics because the source
file
+ // has been deleted here.
+ // The statistics of the mods file can be left unchanged, as it does not
+ // differentiate between sequence or unsequence.
+ FileMetrics.getInstance().deleteTsFile(false,
Collections.singletonList(targetFile));
+ FileMetrics.getInstance()
+ .addTsFile(
+ targetFile.getDatabaseName(),
+ targetFile.getDataRegionId(),
+ targetFile.getTsFileSize(),
+ true,
+ targetFile.getTsFile().getName());
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
index 67c4256843c..d47d7f17277 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
@@ -327,17 +327,21 @@ public class CompactionUtils {
FileMetrics.getInstance().decreaseModFileNum(1);
FileMetrics.getInstance().decreaseModFileSize(resource.getModFile().getSize());
}
- if (!resource.remove()) {
- logger.warn(
- "[Compaction] delete file failed, file path is {}",
- resource.getTsFile().getAbsolutePath());
- } else {
- logger.info("[Compaction] delete file: {}",
resource.getTsFile().getAbsolutePath());
- }
+ deleteTsFileResourceWithoutLock(resource);
}
FileMetrics.getInstance().deleteTsFile(seq, resources);
}
+ public static void deleteTsFileResourceWithoutLock(TsFileResource resource) {
+ if (!resource.remove()) {
+ logger.warn(
+ "[Compaction] delete file failed, file path is {}",
+ resource.getTsFile().getAbsolutePath());
+ } else {
+ logger.info("[Compaction] delete file: {}",
resource.getTsFile().getAbsolutePath());
+ }
+ }
+
public static boolean isDiskHasSpace() {
return isDiskHasSpace(0d);
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java
index 431b9bb9329..efab3d89d74 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java
@@ -22,6 +22,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.compaction.cross;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.AbstractCompactionTask;
@@ -472,6 +473,57 @@ public class InsertionCrossSpaceCompactionTest extends
AbstractCompactionTest {
Assert.assertEquals(4, timestamp);
}
+ @Test
+ public void testInsertionCompactionUpdateFileMetrics() throws IOException {
+ TsFileResource unseqResource1 =
+ generateSingleNonAlignedSeriesFileWithDevices(
+ "2-2-0-0.tsfile", new String[] {"d1"}, new TimeRange[] {new
TimeRange(1, 4)}, false);
+ FileMetrics.getInstance()
+ .addTsFile(
+ unseqResource1.getDatabaseName(),
+ unseqResource1.getDataRegionId(),
+ unseqResource1.getTsFileSize(),
+ false,
+ unseqResource1.getTsFile().getName());
+
+ long seqFileNumBeforeCompaction =
FileMetrics.getInstance().getFileCount(true);
+ long unseqFileNumBeforeCompaction =
FileMetrics.getInstance().getFileCount(false);
+
+ InsertionCrossCompactionTaskResource taskResource = new
InsertionCrossCompactionTaskResource();
+ taskResource.setToInsertUnSeqFile(unseqResource1);
+ InsertionCrossSpaceCompactionTask task =
+ new InsertionCrossSpaceCompactionTask(null, 0, tsFileManager,
taskResource, 0);
+ Assert.assertTrue(task.start());
+ Assert.assertEquals(
+ seqFileNumBeforeCompaction + 1,
FileMetrics.getInstance().getFileCount(true));
+ Assert.assertEquals(
+ unseqFileNumBeforeCompaction - 1,
FileMetrics.getInstance().getFileCount(false));
+
+ // overlap
+ TsFileResource unseqResource2 =
+ generateSingleNonAlignedSeriesFileWithDevices(
+ "3-3-0-0.tsfile", new String[] {"d1"}, new TimeRange[] {new
TimeRange(1, 4)}, false);
+ FileMetrics.getInstance()
+ .addTsFile(
+ unseqResource2.getDatabaseName(),
+ unseqResource2.getDataRegionId(),
+ unseqResource2.getTsFileSize(),
+ false,
+ unseqResource2.getTsFile().getName());
+
+ seqFileNumBeforeCompaction = FileMetrics.getInstance().getFileCount(true);
+ unseqFileNumBeforeCompaction =
FileMetrics.getInstance().getFileCount(false);
+
+ taskResource = new InsertionCrossCompactionTaskResource();
+ taskResource.setToInsertUnSeqFile(unseqResource2);
+ task = new InsertionCrossSpaceCompactionTask(null, 0, tsFileManager,
taskResource, 0);
+ // rollback
+ Assert.assertFalse(task.start());
+ Assert.assertEquals(seqFileNumBeforeCompaction,
FileMetrics.getInstance().getFileCount(true));
+ Assert.assertEquals(
+ unseqFileNumBeforeCompaction,
FileMetrics.getInstance().getFileCount(false));
+ }
+
public TsFileResource generateSingleNonAlignedSeriesFileWithDevices(
String fileName, String[] devices, TimeRange[] timeRanges, boolean seq)
throws IOException {
TsFileResource seqResource1 = createEmptyFileAndResourceWithName(fileName,
seq, 0);
diff --git
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 2116ecbe127..94029f38870 100644
---
a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++
b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1187,7 +1187,7 @@ cross_selector=rewrite
cross_performer=fast
# the selector of inner sequence space compaction task
-# effectiveMode: hod_reload
+# effectiveMode: hot_reload
# Options: size_tiered_single_target,size_tiered_multi_target
inner_seq_selector=size_tiered_multi_target
