This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch object_ttl in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4d73c0dde25f2ee37c7003e7579ea533be9d8d3f Author: shuwenwei <[email protected]> AuthorDate: Fri Dec 19 11:12:20 2025 +0800 add ut --- .../compaction/execute/utils/CompactionUtils.java | 4 +- .../object/ObjectTypeCompactionTest.java | 99 ++++++++++++++++++++-- .../apache/iotdb/commons/conf/CommonConfig.java | 3 +- 3 files changed, 97 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java index 3fff8bd2332..e05dabb8203 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java @@ -539,7 +539,7 @@ public class CompactionUtils { continue; } // buffer 60s to avoid concurrent issues with querying - final long timeLowerBoundInMS = CommonDateTimeUtils.currentTime() - ttlInMS + 60 * 1000; + final long timeLowerBoundInMS = CommonDateTimeUtils.currentTime() - ttlInMS - 60 * 1000; try { recursiveTTLCheckForTableDir( tableDir, 0, tsTable.getTagNum() + 1, !restrictObjectLimit, timeLowerBoundInMS); @@ -554,6 +554,8 @@ public class CompactionUtils { } } + // We try to avoid expensive 'stat' system calls by first checking file name and only performing + // Files.readAttributes when the file may be expired private static void recursiveTTLCheckForTableDir( File currentFile, int depth, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/object/ObjectTypeCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/object/ObjectTypeCompactionTest.java index 4fd7e8c432d..002ed7bd65a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/object/ObjectTypeCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/object/ObjectTypeCompactionTest.java @@ -39,11 +39,13 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.SettleCompactionTask; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; import org.apache.iotdb.db.utils.ObjectTypeUtils; +import com.google.common.io.BaseEncoding; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; @@ -67,8 +69,8 @@ import org.junit.Test; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; @@ -148,6 +150,10 @@ public class ObjectTypeCompactionTest extends AbstractCompactionTest { new ReadChunkCompactionPerformer(), 0); Assert.assertTrue(task.start()); + + Assert.assertTrue(pair1.getRight().exists()); + Assert.assertTrue(pair2.getRight().exists()); + CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG); Assert.assertFalse(pair1.getRight().exists()); Assert.assertTrue(pair2.getRight().exists()); } @@ -169,6 +175,10 @@ public class ObjectTypeCompactionTest extends AbstractCompactionTest { new FastCompactionPerformer(false), 0); Assert.assertTrue(task.start()); + + Assert.assertTrue(pair1.getRight().exists()); + Assert.assertTrue(pair2.getRight().exists()); + CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG); Assert.assertFalse(pair2.getRight().exists()); Assert.assertTrue(pair1.getRight().exists()); } @@ -191,6 +201,9 @@ public class ObjectTypeCompactionTest extends AbstractCompactionTest { 0); Assert.assertTrue(task.start()); Assert.assertTrue(pair1.getRight().exists()); + Assert.assertTrue(pair2.getRight().exists()); + CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG); + Assert.assertTrue(pair1.getRight().exists()); Assert.assertFalse(pair2.getRight().exists()); } @@ -212,6 +225,10 @@ public class ObjectTypeCompactionTest extends AbstractCompactionTest { 1, 0); Assert.assertTrue(task.start()); + + Assert.assertTrue(pair1.getRight().exists()); + Assert.assertTrue(pair2.getRight().exists()); + CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG); Assert.assertFalse(pair2.getRight().exists()); Assert.assertTrue(pair1.getRight().exists()); } @@ -234,10 +251,36 @@ public class ObjectTypeCompactionTest extends AbstractCompactionTest { new FastCompactionPerformer(true), 0); Assert.assertTrue(task.start()); + + Assert.assertTrue(pair1.getRight().exists()); + Assert.assertTrue(pair2.getRight().exists()); + CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG); Assert.assertFalse(pair1.getRight().exists()); Assert.assertTrue(pair2.getRight().exists()); } + @Test + public void testTTLCheck() throws IOException { + config.setRestrictObjectLimit(false); + try { + File file1 = generateBase32ObjectFile(regionDir, System.currentTimeMillis() + 100000, false); + File file2 = generateBase32ObjectFile(regionDir, System.currentTimeMillis() + 200000, true); + File file3 = generateBase32ObjectFile(regionDir, System.currentTimeMillis() - 100000, true); + File file4 = generateBase32ObjectFile(regionDir, System.currentTimeMillis() - 200000, false); + Assert.assertTrue(file1.exists()); + Assert.assertTrue(file2.exists()); + Assert.assertTrue(file3.exists()); + Assert.assertTrue(file4.exists()); + CompactionUtils.executeTTLCheckObjectFilesForTableModel(regionDir, COMPACTION_TEST_SG); + Assert.assertTrue(file1.exists()); + Assert.assertTrue(file2.exists()); + Assert.assertFalse(file3.exists()); + Assert.assertFalse(file4.exists()); + } finally { + config.setRestrictObjectLimit(true); + } + } + @Test public void testPlainObjectBinaryReplaceRegionId() { IObjectPath objectPath = new PlainObjectPath(1, 0, new StringArrayDeviceID("t1.d1"), "s1"); @@ -280,13 +323,32 @@ public class ObjectTypeCompactionTest extends AbstractCompactionTest { private Pair<TsFileResource, File> generateTsFileAndObject( boolean seq, long timestamp, int regionIdInTsFile) throws IOException, WriteProcessException { TsFileResource resource = createEmptyFileAndResource(seq); - Path testFile1 = Files.createTempFile(regionDir.toPath(), "test_", ".bin"); + File dir = + new File( + regionDir.getPath() + + File.separator + + "t1" + + File.separator + + "d1" + + File.separator + + "s1"); + dir.mkdirs(); + File testFile1 = new File(dir, timestamp + ".bin"); byte[] content = new byte[100]; for (int i = 0; i < 100; i++) { content[i] = (byte) i; } - Files.write(testFile1, content); - String relativePathInTsFile = regionIdInTsFile + File.separator + testFile1.toFile().getName(); + Files.write(testFile1.toPath(), content); + String relativePathInTsFile = + regionIdInTsFile + + File.separator + + "t1" + + File.separator + + "d1" + + File.separator + + "s1" + + File.separator + + testFile1.getName(); ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES + relativePathInTsFile.length()); buffer.putLong(100L); buffer.put(BytesUtils.stringToBytes(relativePathInTsFile)); @@ -296,7 +358,8 @@ public class ObjectTypeCompactionTest extends AbstractCompactionTest { writer.getSchema().registerTableSchema(tableSchema); writer.startChunkGroup(deviceID); AlignedChunkWriterImpl alignedChunkWriter = - new AlignedChunkWriterImpl(Arrays.asList(new MeasurementSchema("s1", TSDataType.OBJECT))); + new AlignedChunkWriterImpl( + Collections.singletonList(new MeasurementSchema("s1", TSDataType.OBJECT))); alignedChunkWriter.write(timestamp); alignedChunkWriter.write(timestamp, new Binary(buffer.array()), false); alignedChunkWriter.sealCurrentPage(); @@ -309,6 +372,30 @@ public class ObjectTypeCompactionTest extends AbstractCompactionTest { resource.serialize(); resource.deserialize(); resource.setStatus(TsFileResourceStatus.NORMAL); - return new Pair<>(resource, testFile1.toFile()); + return new Pair<>(resource, testFile1); + } + + private File generateBase32ObjectFile(File regionDir, long timestamp, boolean internalLevel) + throws IOException { + File dir = + new File( + regionDir.getPath() + + File.separator + + toBase32Str("t1") + + (internalLevel ? "" : (File.separator + toBase32Str("d1"))) + + File.separator + + toBase32Str("s1")); + dir.mkdirs(); + File testFile1 = new File(dir, timestamp + ".bin"); + byte[] content = new byte[100]; + for (int i = 0; i < 100; i++) { + content[i] = (byte) i; + } + Files.write(testFile1.toPath(), content); + return testFile1; + } + + private String toBase32Str(String str) { + return BaseEncoding.base32().omitPadding().encode(str.getBytes(StandardCharsets.UTF_8)); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 5057bad83f5..62bc89ec5ae 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -142,8 +142,7 @@ public class CommonConfig { private int ttlRuleCapacity = 1000; /** The interval of ttl check task in each database. The unit is ms. Default is 2 hours. */ - // private long ttlCheckInterval = 7_200_000L; - private long ttlCheckInterval = 10_000L; + private long ttlCheckInterval = 7_200_000L; /** Thrift socket and connection timeout between data node and config node. */ private int cnConnectionTimeoutInMS = (int) TimeUnit.SECONDS.toMillis(60);
