This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/AggPerf in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4c31d460418b7873270199759e187be0dfab237e Author: JackieTien97 <[email protected]> AuthorDate: Mon Oct 28 17:41:28 2024 +0800 make some opt --- .../execution/operator/source/SeriesScanUtil.java | 19 +++++++++++-------- .../plan/planner/TableOperatorGenerator.java | 2 +- .../planner/plan/parameter/SeriesScanOptions.java | 11 ++++++++++- .../db/storageengine/dataregion/DataRegion.java | 2 +- .../dataregion/read/QueryDataSource.java | 20 +++++++++++++++++--- 5 files changed, 40 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 1389666f0b2..15fe07161a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -183,15 +183,18 @@ public class SeriesScanUtil implements Accountable { // differentiate the data of tree model and table model. if (context.isIgnoreAllNullRows()) { ttl = DataNodeTTLCache.getInstance().getTTLForTree(deviceID); + scanOptions.setTTL(ttl); } else { - String databaseName = dataSource.getDatabaseName(); - ttl = - databaseName == null - ? Long.MAX_VALUE - : DataNodeTTLCache.getInstance() - .getTTLForTable(databaseName, deviceID.getTableName()); - } - scanOptions.setTTL(ttl); + if (scanOptions.timeFilterNeedUpdatedByTll()) { + String databaseName = dataSource.getDatabaseName(); + ttl = + databaseName == null + ? Long.MAX_VALUE + : DataNodeTTLCache.getInstance() + .getTTLForTable(databaseName, deviceID.getTableName()); + scanOptions.setTTL(ttl); + } + } // init file index orderUtils.setCurSeqFileIndex(dataSource); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 56d6868a24b..d3150c2f731 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -1427,7 +1427,7 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution groupByChannels, aggregatorBuilder.build(), node.getStep(), - 10_000, + 64, Long.MAX_VALUE, false, Long.MAX_VALUE); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java index dea62b9cf05..25853c9c547 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/SeriesScanOptions.java @@ -32,11 +32,14 @@ import org.apache.tsfile.read.reader.series.PaginationController; import java.util.Collections; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; public class SeriesScanOptions { private Filter globalTimeFilter; + private final AtomicBoolean timeFilterUpdatedByTll = new AtomicBoolean(false); + private final Filter pushDownFilter; private final long pushDownLimit; @@ -97,8 +100,14 @@ public class SeriesScanOptions { } } + public boolean timeFilterNeedUpdatedByTll() { + return !timeFilterUpdatedByTll.get(); + } + public void setTTL(long dataTTL) { - this.globalTimeFilter = updateFilterUsingTTL(globalTimeFilter, dataTTL); + if (timeFilterUpdatedByTll.compareAndSet(false, true)) { + this.globalTimeFilter = updateFilterUsingTTL(globalTimeFilter, dataTTL); + } } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 16654ea8b16..e05ca6222eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -2012,7 +2012,7 @@ public class DataRegion implements IDataRegionForQuery { QUERY_RESOURCE_METRIC_SET.recordQueryResourceNum(SEQUENCE_TSFILE, seqResources.size()); QUERY_RESOURCE_METRIC_SET.recordQueryResourceNum(UNSEQUENCE_TSFILE, unseqResources.size()); - return new QueryDataSource(seqResources, unseqResources); + return new QueryDataSource(seqResources, unseqResources, databaseName); } catch (MetadataException e) { throw new QueryProcessException(e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java index dc56ad11a12..4748cc20e58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSource.java @@ -64,6 +64,8 @@ public class QueryDataSource implements IQueryDataSource { /* The traversal order of unseqResources (different for each device) */ private int[] unSeqFileOrderIndex; + private String databaseName = null; + private static final Comparator<Long> descendingComparator = (o1, o2) -> Long.compare(o2, o1); public QueryDataSource(List<TsFileResource> seqResources, List<TsFileResource> unseqResources) { @@ -71,12 +73,20 @@ public class QueryDataSource implements IQueryDataSource { this.unseqResources = unseqResources; } + public QueryDataSource( + List<TsFileResource> seqResources, List<TsFileResource> unseqResources, String databaseName) { + this.seqResources = seqResources; + this.unseqResources = unseqResources; + this.databaseName = databaseName; + } + // used for compaction, because in compaction task(unlike query, each QueryDataSource only serve // for one series), we will reuse this object for multi series public QueryDataSource(QueryDataSource other) { this.seqResources = other.seqResources; this.unseqResources = other.unseqResources; this.unSeqFileOrderIndex = other.unSeqFileOrderIndex; + this.databaseName = other.databaseName; } public List<TsFileResource> getSeqResources() { @@ -89,7 +99,8 @@ public class QueryDataSource implements IQueryDataSource { @Override public IQueryDataSource clone() { - QueryDataSource queryDataSource = new QueryDataSource(getSeqResources(), getUnseqResources()); + QueryDataSource queryDataSource = + new QueryDataSource(getSeqResources(), getUnseqResources(), databaseName); queryDataSource.setSingleDevice(isSingleDevice()); return queryDataSource; } @@ -226,7 +237,10 @@ public class QueryDataSource implements IQueryDataSource { } public String getDatabaseName() { - List<TsFileResource> resources = !seqResources.isEmpty() ? seqResources : unseqResources; - return resources.isEmpty() ? null : resources.get(0).getDatabaseName(); + if (databaseName == null) { + List<TsFileResource> resources = !seqResources.isEmpty() ? seqResources : unseqResources; + databaseName = resources.isEmpty() ? null : resources.get(0).getDatabaseName(); + } + return databaseName; } }
