This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch skipTTLCheckInSomeCases in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d6157b5e5842fc8db117ddbdb280e14f4b5bfd9a Author: shuwenwei <[email protected]> AuthorDate: Wed Aug 6 14:47:27 2025 +0800 Skip TTL check when there is no ttl in the data region and no mods file --- .../analyze/cache/schema/DataNodeTTLCache.java | 9 ++++ .../db/storageengine/dataregion/DataRegion.java | 49 +++++++++++++++++++++- .../iotdb/db/storageengine/dataregion/TTLTest.java | 48 +++++++++++++++++++++ .../apache/iotdb/commons/schema/ttl/TTLCache.java | 42 +++++++++++++++++++ 4 files changed, 147 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java index 5e0f3f01a2b..58433616a20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java @@ -109,6 +109,15 @@ public class DataNodeTTLCache { } } + public boolean dataInDatabaseMayHaveTTL(String db) { + lock.readLock().lock(); + try { + return treeModelTTLCache.dataInDatabaseMayHaveTTL(db); + } finally { + lock.readLock().unlock(); + } + } + /** * Get ttl of one specific path node without time precision conversion. If this node does not set * ttl, then return -1. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 04bed96d2ff..617028b50f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -31,6 +31,7 @@ import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.path.IFullPath; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.table.InformationSchema; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.service.metric.MetricService; @@ -42,6 +43,8 @@ import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp; +import org.apache.iotdb.confignode.rpc.thrift.TTableInfo; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -62,6 +65,7 @@ import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResource.Status; import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager; import org.apache.iotdb.db.pipe.consensus.deletion.persist.PageCacheDeletionBuffer; import org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener; +import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.queryengine.common.DeviceContext; @@ -201,6 +205,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR; import static org.apache.iotdb.commons.utils.PathUtils.isTableModelDatabase; @@ -2904,9 +2909,12 @@ public class DataRegion implements IDataRegionForQuery { // wait until success Thread.sleep(500); } - logger.info("[TTL] {}-{} Start ttl checking.", databaseName, dataRegionId); int trySubmitCount = 0; try { + if (skipCurrentTTLAndModificationCheck()) { + return 0; + } + logger.info("[TTL] {}-{} Start ttl and modification checking.", databaseName, dataRegionId); CompactionScheduleContext context = new CompactionScheduleContext(); List<Long> timePartitions = new ArrayList<>(tsFileManager.getTimePartitions()); // Sort the time partition from smallest to largest @@ -2942,6 +2950,45 @@ public class DataRegion implements IDataRegionForQuery { return trySubmitCount; } + private boolean skipCurrentTTLAndModificationCheck() { + if (this.databaseName.equals(InformationSchema.INFORMATION_DATABASE)) { + return true; + } + for (Long timePartition : getTimePartitions()) { + List<TsFileResource> seqFiles = tsFileManager.getTsFileListSnapshot(timePartition, true); + List<TsFileResource> unseqFiles = tsFileManager.getTsFileListSnapshot(timePartition, false); + boolean modFileExists = + Stream.concat(seqFiles.stream(), unseqFiles.stream()) + .anyMatch(TsFileResource::anyModFileExists); + if (modFileExists) { + return false; + } + } + boolean isTableModel = isTableModelDatabase(this.databaseName); + if (isTableModel) { + try (final ConfigNodeClient configNodeClient = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + TShowTableResp resp = configNodeClient.showTables(databaseName, false); + for (TTableInfo tTableInfo : resp.getTableInfoList()) { + String ttl = tTableInfo.getTTL(); + if (ttl == null || ttl.equals(IoTDBConstant.TTL_INFINITE)) { + continue; + } + long ttlValue = Long.parseLong(ttl); + if (ttlValue < 0 || ttlValue == Long.MAX_VALUE) { + continue; + } + return false; + } + return true; + } catch (Exception ignored) { + return false; + } + } else { + return !DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL(databaseName); + } + } + protected int[] executeInsertionCompaction( List<Long> timePartitions, CompactionScheduleContext context) throws InterruptedException { int[] trySubmitCountOfTimePartitions = new int[timePartitions.size()]; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java index 9bc0d73311b..41bcce8e2ad 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java @@ -60,6 +60,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -259,6 +260,53 @@ public class TTLTest { assertTrue(cnt == 0); } + @Test + public void testTTLRead2() throws IllegalPathException { + DataNodeTTLCache.getInstance().setTTLForTree("root.test1.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1")); + DataNodeTTLCache.getInstance().clearAllTTLForTree(); + + DataNodeTTLCache.getInstance().setTTLForTree("root.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1")); + DataNodeTTLCache.getInstance().clearAllTTLForTree(); + + DataNodeTTLCache.getInstance().setTTLForTree("root.test1.d1.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1")); + DataNodeTTLCache.getInstance().clearAllTTLForTree(); + + DataNodeTTLCache.getInstance().setTTLForTree("root.test2.**", 500); + Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1")); + DataNodeTTLCache.getInstance().clearAllTTLForTree(); + + DataNodeTTLCache.getInstance().setTTLForTree("root.test2.d1.**", 500); + Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1")); + DataNodeTTLCache.getInstance().clearAllTTLForTree(); + + DataNodeTTLCache.getInstance().setTTLForTree("root.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTLForTree(); + + DataNodeTTLCache.getInstance().setTTLForTree("root.test.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTLForTree(); + + DataNodeTTLCache.getInstance().setTTLForTree("root.test1.**", 500); + Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTLForTree(); + + DataNodeTTLCache.getInstance().setTTLForTree("root.test.sg1.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTLForTree(); + + DataNodeTTLCache.getInstance().setTTLForTree("root.test.sg2.**", 500); + Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTLForTree(); + + DataNodeTTLCache.getInstance().setTTLForTree("root.test.sg1.d1.**", 500); + Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1")); + DataNodeTTLCache.getInstance().clearAllTTLForTree(); + } + private NonAlignedFullPath mockMeasurementPath() { return new NonAlignedFullPath( IDeviceID.Factory.DEFAULT_FACTORY.create(sg1Device), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java index bfb67069ccc..d538c3958aa 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java @@ -199,6 +199,48 @@ public class TTLCache { return maxTTL; } + public boolean dataInDatabaseMayHaveTTL(String database) { + String[] nodeNames = database.split("\\."); + CacheNode current = ttlCacheTree; + for (String nodeName : nodeNames) { + if (hasValidTTLOnCurrentLevel(current)) { + return true; + } + if (nodeName.equals("root")) { + continue; + } + current = current.getChild(nodeName); + if (current == null) { + return false; + } + } + + if (hasValidTTLOnCurrentLevel(current)) { + return true; + } + + Queue<CacheNode> queue = new LinkedList<>(); + queue.add(current); + while (!queue.isEmpty()) { + current = queue.poll(); + for (CacheNode child : current.getChildren().values()) { + if (child.ttl >= 0 && child.ttl != Long.MAX_VALUE) { + return true; + } + queue.add(child); + } + } + return false; + } + + private boolean hasValidTTLOnCurrentLevel(CacheNode current) { + if (current.ttl >= 0 && current.ttl != Long.MAX_VALUE) { + return true; + } + CacheNode wildcardChild = current.getChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD); + return wildcardChild != null && wildcardChild.ttl >= 0 && wildcardChild.ttl != Long.MAX_VALUE; + } + /** * @return key is path contains wildcard between each node */
