This is an automated email from the ASF dual-hosted git repository. ycycse pushed a commit to branch ttlInActive in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0ac380e4a3dd31fc01a5a7546c7ea07002e117b3 Author: YangCaiyin <[email protected]> AuthorDate: Tue Jul 9 23:36:28 2024 +0800 feat: support new filter for ttl --- .../source/ActiveDeviceRegionScanOperator.java | 1 - .../source/RegionScanForActiveDeviceUtil.java | 12 ++++- .../iotdb/db/utils/TimeFilterForDeviceTTL.java | 54 ++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java index 6437fe673fa..e74d1c675df 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java @@ -91,7 +91,6 @@ public class ActiveDeviceRegionScanOperator extends AbstractRegionScanDataSource int templateId = deviceContext.getTemplateId(); // TODO: use IDeviceID interface to get ttl long ttl = DataNodeTTLCache.getInstance().getTTL(deviceID); - // TODO: make it more readable, like "30 days" or "10 hours" String ttlStr = ttl == Long.MAX_VALUE ? IoTDBConstant.TTL_INFINITE : String.valueOf(ttl); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java index 11faa55f394..cb66e0aeee6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java @@ -20,7 +20,9 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.db.queryengine.common.DeviceContext; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractChunkOffset; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractDeviceChunkMetaData; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.DeviceStartEndTime; @@ -77,7 +79,7 @@ public class RegionScanForActiveDeviceUtil extends AbstractRegionScanForActiveDa while (iterator.hasNext()) { DeviceStartEndTime deviceStartEndTime = iterator.next(); IDeviceID deviceID = deviceStartEndTime.getDevicePath(); - long startTime = deviceStartEndTime.getStartTime(); + long startTime = getStartTimeConsideringTTL(deviceID, deviceStartEndTime.getStartTime()); long endTime = deviceStartEndTime.getEndTime(); // If this device has already been removed by another TsFileHandle, we should skip it. // If the time range is filtered, the devicePath is not active in this time range. @@ -100,6 +102,14 @@ public class RegionScanForActiveDeviceUtil extends AbstractRegionScanForActiveDa return true; } + private long getStartTimeConsideringTTL(IDeviceID deviceID, long originalStartTime){ + long ttl = DataNodeTTLCache.getInstance().getTTL(deviceID); + if(ttl != Long.MAX_VALUE){ + return Math.max(originalStartTime, CommonDateTimeUtils.currentTime() - ttl); + } + return originalStartTime; + } + @Override public void processDeviceChunkMetadata(AbstractDeviceChunkMetaData deviceChunkMetaData) throws IllegalPathException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java new file mode 100644 index 00000000000..8e3505a871b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java @@ -0,0 +1,54 @@ +package org.apache.iotdb.db.utils; + +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.filter.basic.Filter; + +import java.util.Map; + +public class TimeFilterForDeviceTTL { + + private Filter timeFilter; + + private Map<IDeviceID, Long> ttlCached; + + public TimeFilterForDeviceTTL(Filter timeFilter) { + this.timeFilter = timeFilter; + } + + public boolean satisfyStartEndTIme(long startTime, long endTime, IDeviceID deviceID){ + long ttl = getTTL(deviceID); + if(ttl != Long.MAX_VALUE){ + long validStartTime = CommonDateTimeUtils.currentTime() - ttl; + if(validStartTime > endTime){ + return false; + } + return timeFilter.satisfyStartEndTime(validStartTime, endTime); + } + return timeFilter.satisfyStartEndTime(startTime, endTime); + } + + public boolean satisfy(long time, IDeviceID deviceID){ + long ttl = getTTL(deviceID); + if(ttl != Long.MAX_VALUE) { + long validStartTime = CommonDateTimeUtils.currentTime() - ttl; + if(validStartTime > time){ + return false; + } + return timeFilter.satisfy(validStartTime, null); + } + return timeFilter.satisfy(time, null); + } + + private long getTTL(IDeviceID deviceID){ + if(ttlCached.containsKey(deviceID)){ + return ttlCached.get(deviceID); + } + long ttl = DataNodeTTLCache.getInstance().getTTL(deviceID); + ttlCached.put(deviceID, ttl); + return ttl; + } + + +}
