This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a commit to branch ttlInActive
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/ttlInActive by this push:
new 7f1cba5f326 feat: use TimeFilterForDeviceTTL in regionScanUtil
7f1cba5f326 is described below
commit 7f1cba5f326903a3d90e567a3f2e27f71c351db5
Author: ycycse <[email protected]>
AuthorDate: Wed Jul 10 01:04:28 2024 +0800
feat: use TimeFilterForDeviceTTL in regionScanUtil
---
.../AbstractRegionScanForActiveDataUtil.java | 7 ++-
.../source/RegionScanForActiveDeviceUtil.java | 16 +----
.../source/RegionScanForActiveTimeSeriesUtil.java | 4 +-
.../iotdb/db/utils/TimeFilterForDeviceTTL.java | 69 +++++++++++-----------
4 files changed, 43 insertions(+), 53 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractRegionScanForActiveDataUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractRegionScanForActiveDataUtil.java
index 530e5ce06db..8d43f142412 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractRegionScanForActiveDataUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractRegionScanForActiveDataUtil.java
@@ -26,6 +26,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.read.filescan.IChunkHandle;
import
org.apache.iotdb.db.storageengine.dataregion.read.filescan.IFileScanHandle;
import
org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractChunkOffset;
import
org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractDeviceChunkMetaData;
+import org.apache.iotdb.db.utils.TimeFilterForDeviceTTL;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.statistics.Statistics;
@@ -49,7 +50,7 @@ public abstract class AbstractRegionScanForActiveDataUtil
implements Accountable
+ RamUsageEstimator.shallowSizeOfInstance(TimeFilter.class);
protected QueryDataSourceForRegionScan queryDataSource;
- protected final Filter timeFilter;
+ protected final TimeFilterForDeviceTTL timeFilter;
protected final List<AbstractChunkOffset> chunkToBeScanned = new
ArrayList<>();
protected final List<Statistics<? extends Serializable>> chunkStatistics =
new ArrayList<>();
@@ -60,7 +61,7 @@ public abstract class AbstractRegionScanForActiveDataUtil
implements Accountable
protected IChunkHandle currentChunkHandle = null;
protected AbstractRegionScanForActiveDataUtil(Filter timeFilter) {
- this.timeFilter = timeFilter;
+ this.timeFilter = new TimeFilterForDeviceTTL(timeFilter);
}
public void initQueryDataSource(IQueryDataSource dataSource) {
@@ -146,7 +147,7 @@ public abstract class AbstractRegionScanForActiveDataUtil
implements Accountable
IDeviceID curDevice = currentChunkHandle.getDeviceID();
String curMeasurement = currentChunkHandle.getMeasurement();
long[] pageStatistics = currentChunkHandle.getPageStatisticsTime();
- if (!timeFilter.satisfyStartEndTime(pageStatistics[0], pageStatistics[1]))
{
+ if (!timeFilter.satisfyStartEndTime(pageStatistics[0], pageStatistics[1],
curDevice)) {
// All the data in current page is not valid, just skip.
currentChunkHandle.skipCurrentPage();
return true;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java
index cb66e0aeee6..63bcbe35187 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java
@@ -20,9 +20,7 @@
package org.apache.iotdb.db.queryengine.execution.operator.source;
import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.db.queryengine.common.DeviceContext;
-import
org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
import
org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractChunkOffset;
import
org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractDeviceChunkMetaData;
import
org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.DeviceStartEndTime;
@@ -79,12 +77,12 @@ public class RegionScanForActiveDeviceUtil extends
AbstractRegionScanForActiveDa
while (iterator.hasNext()) {
DeviceStartEndTime deviceStartEndTime = iterator.next();
IDeviceID deviceID = deviceStartEndTime.getDevicePath();
- long startTime = getStartTimeConsideringTTL(deviceID,
deviceStartEndTime.getStartTime());
+ long startTime = deviceStartEndTime.getStartTime();
long endTime = deviceStartEndTime.getEndTime();
// If this device has already been removed by another TsFileHandle, we
should skip it.
// If the time range is filtered, the devicePath is not active in this
time range.
if (!targetDevices.containsKey(deviceID)
- || (endTime >= 0 && !timeFilter.satisfyStartEndTime(startTime,
endTime))) {
+ || (endTime >= 0 && !timeFilter.satisfyStartEndTime(startTime,
endTime, deviceID))) {
continue;
}
@@ -102,14 +100,6 @@ public class RegionScanForActiveDeviceUtil extends
AbstractRegionScanForActiveDa
return true;
}
- private long getStartTimeConsideringTTL(IDeviceID deviceID, long
originalStartTime){
- long ttl = DataNodeTTLCache.getInstance().getTTL(deviceID);
- if(ttl != Long.MAX_VALUE){
- return Math.max(originalStartTime, CommonDateTimeUtils.currentTime() -
ttl);
- }
- return originalStartTime;
- }
-
@Override
public void processDeviceChunkMetadata(AbstractDeviceChunkMetaData
deviceChunkMetaData)
throws IllegalPathException {
@@ -145,7 +135,7 @@ public class RegionScanForActiveDeviceUtil extends
AbstractRegionScanForActiveDa
IChunkMetadata valueChunkMetaData =
deviceChunkMetaData.nextValueChunkMetadata();
long startTime = valueChunkMetaData.getStartTime();
long endTime = valueChunkMetaData.getEndTime();
- if (!timeFilter.satisfyStartEndTime(startTime, endTime)) {
+ if (!timeFilter.satisfyStartEndTime(startTime, endTime, deviceID)) {
continue;
}
String measurement = valueChunkMetaData.getMeasurementUid();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java
index 61c7028ab21..c53d4f388db 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java
@@ -73,7 +73,7 @@ public class RegionScanForActiveTimeSeriesUtil extends
AbstractRegionScanForActi
long startTime = deviceStartEndTime.getStartTime();
long endTime = deviceStartEndTime.getEndTime();
if (!targetTimeseries.containsKey(deviceID)
- || (endTime >= 0 && !timeFilter.satisfyStartEndTime(startTime,
endTime))) {
+ || (endTime >= 0 && !timeFilter.satisfyStartEndTime(startTime,
endTime, deviceID))) {
continue;
}
@@ -132,7 +132,7 @@ public class RegionScanForActiveTimeSeriesUtil extends
AbstractRegionScanForActi
Set<String> measurementForCurrentTsFile =
timeSeriesForCurrentTsFile.get(deviceID);
if (!(measurementForCurrentTsFile != null
&& measurementForCurrentTsFile.contains(measurementId))
- || !timeFilter.satisfyStartEndTime(startTime, endTime)) {
+ || !timeFilter.satisfyStartEndTime(startTime, endTime, deviceID)) {
continue;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java
index 8e3505a871b..f9c579dd7e7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java
@@ -2,6 +2,7 @@ package org.apache.iotdb.db.utils;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import
org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
+
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.filter.basic.Filter;
@@ -9,46 +10,44 @@ import java.util.Map;
public class TimeFilterForDeviceTTL {
- private Filter timeFilter;
+ private final Filter timeFilter;
- private Map<IDeviceID, Long> ttlCached;
+ private Map<IDeviceID, Long> ttlCached;
- public TimeFilterForDeviceTTL(Filter timeFilter) {
- this.timeFilter = timeFilter;
- }
+ public TimeFilterForDeviceTTL(Filter timeFilter) {
+ this.timeFilter = timeFilter;
+ }
- public boolean satisfyStartEndTIme(long startTime, long endTime, IDeviceID
deviceID){
- long ttl = getTTL(deviceID);
- if(ttl != Long.MAX_VALUE){
- long validStartTime = CommonDateTimeUtils.currentTime() - ttl;
- if(validStartTime > endTime){
- return false;
- }
- return timeFilter.satisfyStartEndTime(validStartTime, endTime);
- }
- return timeFilter.satisfyStartEndTime(startTime, endTime);
+ public boolean satisfyStartEndTime(long startTime, long endTime, IDeviceID
deviceID) {
+ long ttl = getTTL(deviceID);
+ if (ttl != Long.MAX_VALUE) {
+ long validStartTime = CommonDateTimeUtils.currentTime() - ttl;
+ if (validStartTime > endTime) {
+ return false;
+ }
+ return timeFilter.satisfyStartEndTime(validStartTime, endTime);
}
-
- public boolean satisfy(long time, IDeviceID deviceID){
- long ttl = getTTL(deviceID);
- if(ttl != Long.MAX_VALUE) {
- long validStartTime = CommonDateTimeUtils.currentTime() - ttl;
- if(validStartTime > time){
- return false;
- }
- return timeFilter.satisfy(validStartTime, null);
- }
- return timeFilter.satisfy(time, null);
+ return timeFilter.satisfyStartEndTime(startTime, endTime);
+ }
+
+ public boolean satisfy(long time, IDeviceID deviceID) {
+ long ttl = getTTL(deviceID);
+ if (ttl != Long.MAX_VALUE) {
+ long validStartTime = CommonDateTimeUtils.currentTime() - ttl;
+ if (validStartTime > time) {
+ return false;
+ }
+ return timeFilter.satisfy(validStartTime, null);
}
+ return timeFilter.satisfy(time, null);
+ }
- private long getTTL(IDeviceID deviceID){
- if(ttlCached.containsKey(deviceID)){
- return ttlCached.get(deviceID);
- }
- long ttl = DataNodeTTLCache.getInstance().getTTL(deviceID);
- ttlCached.put(deviceID, ttl);
- return ttl;
+ private long getTTL(IDeviceID deviceID) {
+ if (ttlCached.containsKey(deviceID)) {
+ return ttlCached.get(deviceID);
}
-
-
+ long ttl = DataNodeTTLCache.getInstance().getTTL(deviceID);
+ ttlCached.put(deviceID, ttl);
+ return ttl;
+ }
}
