This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 77f1dcc7ad7 Skip TTL check when there is no TTL and no mods file in
the data region (#16110)
77f1dcc7ad7 is described below
commit 77f1dcc7ad755707680de4b3588eefbd1536d842
Author: shuwenwei <[email protected]>
AuthorDate: Thu Aug 7 10:41:10 2025 +0800
Skip TTL check when there is no TTL and no mods file in the data region
(#16110)
* Skip TTL check when there is no ttl in the data region and no mods file
* check back quote
---
.../iotdb/confignode/persistence/TTLInfo.java | 2 +
.../analyze/cache/schema/DataNodeTTLCache.java | 9 ++++
.../db/storageengine/dataregion/DataRegion.java | 53 +++++++++++++++++++-
.../iotdb/db/storageengine/dataregion/TTLTest.java | 52 ++++++++++++++++++++
.../apache/iotdb/commons/schema/ttl/TTLCache.java | 56 ++++++++++++++++++++--
5 files changed, 168 insertions(+), 4 deletions(-)
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java
index edc00411d18..0d07f9414d4 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java
@@ -169,6 +169,8 @@ public class TTLInfo implements SnapshotProcessor {
lock.readLock().lock();
try {
return ttlCache.getDatabaseMaxTTL(database);
+ } catch (IllegalPathException e) {
+ return TTLCache.NULL_TTL;
} finally {
lock.readLock().unlock();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
index 5e0f3f01a2b..b63d0400dbc 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
@@ -109,6 +109,15 @@ public class DataNodeTTLCache {
}
}
+ public boolean dataInDatabaseMayHaveTTL(String db) throws
IllegalPathException {
+ lock.readLock().lock();
+ try {
+ return treeModelTTLCache.dataInDatabaseMayHaveTTL(db);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
/**
* Get ttl of one specific path node without time precision conversion. If
this node does not set
* ttl, then return -1.
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index b9493415a33..329728ab130 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.commons.file.SystemFileFactory;
import org.apache.iotdb.commons.path.IFullPath;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.schema.SchemaConstant;
+import org.apache.iotdb.commons.schema.table.InformationSchema;
import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil;
import org.apache.iotdb.commons.service.metric.MetricService;
@@ -42,6 +43,8 @@ import org.apache.iotdb.commons.utils.RetryUtils;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp;
+import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp;
+import org.apache.iotdb.confignode.rpc.thrift.TTableInfo;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -62,6 +65,7 @@ import
org.apache.iotdb.db.pipe.consensus.deletion.DeletionResource.Status;
import org.apache.iotdb.db.pipe.consensus.deletion.DeletionResourceManager;
import
org.apache.iotdb.db.pipe.consensus.deletion.persist.PageCacheDeletionBuffer;
import
org.apache.iotdb.db.pipe.source.dataregion.realtime.listener.PipeInsertionDataNodeListener;
+import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
import org.apache.iotdb.db.queryengine.common.DeviceContext;
@@ -201,6 +205,7 @@ import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
import static org.apache.iotdb.commons.utils.PathUtils.isTableModelDatabase;
@@ -2904,9 +2909,12 @@ public class DataRegion implements IDataRegionForQuery {
// wait until success
Thread.sleep(500);
}
- logger.info("[TTL] {}-{} Start ttl checking.", databaseName, dataRegionId);
int trySubmitCount = 0;
try {
+ if (skipCurrentTTLAndModificationCheck()) {
+ return 0;
+ }
+ logger.info("[TTL] {}-{} Start ttl and modification checking.",
databaseName, dataRegionId);
CompactionScheduleContext context = new CompactionScheduleContext();
List<Long> timePartitions = new
ArrayList<>(tsFileManager.getTimePartitions());
// Sort the time partition from smallest to largest
@@ -2942,6 +2950,49 @@ public class DataRegion implements IDataRegionForQuery {
return trySubmitCount;
}
+ private boolean skipCurrentTTLAndModificationCheck() {
+ if (this.databaseName.equals(InformationSchema.INFORMATION_DATABASE)) {
+ return true;
+ }
+ for (Long timePartition : getTimePartitions()) {
+ List<TsFileResource> seqFiles =
tsFileManager.getTsFileListSnapshot(timePartition, true);
+ List<TsFileResource> unseqFiles =
tsFileManager.getTsFileListSnapshot(timePartition, false);
+ boolean modFileExists =
+ Stream.concat(seqFiles.stream(), unseqFiles.stream())
+ .anyMatch(TsFileResource::anyModFileExists);
+ if (modFileExists) {
+ return false;
+ }
+ }
+ boolean isTableModel = isTableModelDatabase(this.databaseName);
+ if (isTableModel) {
+ try (final ConfigNodeClient configNodeClient =
+
ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID))
{
+ TShowTableResp resp = configNodeClient.showTables(databaseName, false);
+ for (TTableInfo tTableInfo : resp.getTableInfoList()) {
+ String ttl = tTableInfo.getTTL();
+ if (ttl == null || ttl.equals(IoTDBConstant.TTL_INFINITE)) {
+ continue;
+ }
+ long ttlValue = Long.parseLong(ttl);
+ if (ttlValue < 0 || ttlValue == Long.MAX_VALUE) {
+ continue;
+ }
+ return false;
+ }
+ return true;
+ } catch (Exception ignored) {
+ return false;
+ }
+ } else {
+ try {
+ return
!DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL(databaseName);
+ } catch (Exception ignored) {
+ return false;
+ }
+ }
+ }
+
protected int[] executeInsertionCompaction(
List<Long> timePartitions, CompactionScheduleContext context) throws
InterruptedException {
int[] trySubmitCountOfTimePartitions = new int[timePartitions.size()];
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java
index 9bc0d73311b..afd7ed608ea 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TTLTest.java
@@ -60,6 +60,7 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.common.block.TsBlock;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -259,6 +260,57 @@ public class TTLTest {
assertTrue(cnt == 0);
}
+ @Test
+ public void testTTLRead2() throws IllegalPathException {
+ DataNodeTTLCache.getInstance().setTTLForTree("root.test1.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+
+ DataNodeTTLCache.getInstance().setTTLForTree("root.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+
+ DataNodeTTLCache.getInstance().setTTLForTree("root.test1.d1.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+
+ DataNodeTTLCache.getInstance().setTTLForTree("root.test2.**", 500);
+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+
+ DataNodeTTLCache.getInstance().setTTLForTree("root.test2.d1.**", 500);
+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test1"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+
+ DataNodeTTLCache.getInstance().setTTLForTree("root.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+
+ DataNodeTTLCache.getInstance().setTTLForTree("root.test.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+
+ DataNodeTTLCache.getInstance().setTTLForTree("root.test1.**", 500);
+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+
+ DataNodeTTLCache.getInstance().setTTLForTree("root.test.sg1.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+
+ DataNodeTTLCache.getInstance().setTTLForTree("root.test.sg2.**", 500);
+
Assert.assertFalse(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+
+ DataNodeTTLCache.getInstance().setTTLForTree("root.test.sg1.d1.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.test.sg1"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+
+ DataNodeTTLCache.getInstance().setTTLForTree("root.`1.1`.**", 500);
+
Assert.assertTrue(DataNodeTTLCache.getInstance().dataInDatabaseMayHaveTTL("root.`1.1`"));
+ DataNodeTTLCache.getInstance().clearAllTTLForTree();
+ }
+
private NonAlignedFullPath mockMeasurementPath() {
return new NonAlignedFullPath(
IDeviceID.Factory.DEFAULT_FACTORY.create(sg1Device),
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java
index bfb67069ccc..9bed883a275 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.commons.utils.StatusUtils;
import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import javax.annotation.concurrent.NotThreadSafe;
@@ -181,7 +182,7 @@ public class TTLCache {
* @return the maximum ttl of the subtree of the corresponding database.
return NULL_TTL if the
* TTL is not set or the database does not exist.
*/
- public long getDatabaseMaxTTL(String database) {
+ public long getDatabaseMaxTTL(String database) throws IllegalPathException {
CacheNode node = ttlCacheTree.searchChild(database);
if (node == null) {
return NULL_TTL;
@@ -199,6 +200,55 @@ public class TTLCache {
return maxTTL;
}
+ public boolean dataInDatabaseMayHaveTTL(String database) throws
IllegalPathException {
+ String[] nodeNames = split(database);
+ CacheNode current = ttlCacheTree;
+ for (String nodeName : nodeNames) {
+ if (hasValidTTLOnCurrentLevel(current)) {
+ return true;
+ }
+ if (nodeName.equals("root")) {
+ continue;
+ }
+ current = current.getChild(nodeName);
+ if (current == null) {
+ return false;
+ }
+ }
+
+ if (hasValidTTLOnCurrentLevel(current)) {
+ return true;
+ }
+
+ Queue<CacheNode> queue = new LinkedList<>();
+ queue.add(current);
+ while (!queue.isEmpty()) {
+ current = queue.poll();
+ for (CacheNode child : current.getChildren().values()) {
+ if (child.ttl >= 0 && child.ttl != Long.MAX_VALUE) {
+ return true;
+ }
+ queue.add(child);
+ }
+ }
+ return false;
+ }
+
+ private boolean hasValidTTLOnCurrentLevel(CacheNode current) {
+ if (current.ttl >= 0 && current.ttl != Long.MAX_VALUE) {
+ return true;
+ }
+ CacheNode wildcardChild =
current.getChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD);
+ return wildcardChild != null && wildcardChild.ttl >= 0 &&
wildcardChild.ttl != Long.MAX_VALUE;
+ }
+
+ private static String[] split(String path) throws IllegalPathException {
+ if (!path.contains(TsFileConstant.BACK_QUOTE_STRING)) {
+ return path.split(TsFileConstant.PATH_SEPARATER_NO_REGEX);
+ }
+ return PathUtils.splitPathToDetachedNodes(path);
+ }
+
/**
* @return key is path contains wildcard between each node
*/
@@ -306,8 +356,8 @@ public class TTLCache {
* @param name the name corresponding to the child node, use '.' to
separate each node
* @return the child node if it exists, otherwise return null
*/
- public CacheNode searchChild(String name) {
- String[] nodeNames = name.split("\\.");
+ public CacheNode searchChild(String name) throws IllegalPathException {
+ String[] nodeNames = split(name);
CacheNode current = this;
for (String nodeName : nodeNames) {
if (nodeName.equals("root")) {
