This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch read_tsfile_table_function in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 539196705470844b38af998c0ead4a00ecd24494 Author: shuwenwei <[email protected]> AuthorDate: Wed Jun 3 14:51:29 2026 +0800 fix --- .../execution/driver/DataDriverContext.java | 16 --------------- .../fragment/FragmentInstanceContext.java | 8 ++++++-- .../relational/MultiTsFileResourceIterator.java | 5 ++--- .../OrderedExternalTsFileTableScanOperator.java | 24 ++++++++++------------ .../planner/DataNodeTableOperatorGenerator.java | 2 +- .../plan/planner/LocalExecutionPlanner.java | 16 --------------- .../plan/planner/plan/node/PlanGraphPrinter.java | 21 +++++++++++++++++++ .../ConvertSchemaPredicateToFilterVisitor.java | 21 +++++++++++++++++-- 8 files changed, 60 insertions(+), 53 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java index a2892b6fec0..f895859d1ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java @@ -41,7 +41,6 @@ public class DataDriverContext extends DriverContext { private List<IFullPath> paths; private QueryDataSourceType queryDataSourceType = null; private Map<IDeviceID, DeviceContext> deviceIDToContext; - private List<String> externalTsFilePaths; // it will be set to null, after QueryDataSource being inited private List<DataSourceOperator> sourceOperators; @@ -50,7 +49,6 @@ public class DataDriverContext extends DriverContext { this.paths = new ArrayList<>(); this.sourceOperators = new ArrayList<>(); this.deviceIDToContext = null; - this.externalTsFilePaths = null; } public DataDriverContext(DataDriverContext parentContext, int pipelineId) { @@ -58,7 +56,6 @@ public class DataDriverContext extends DriverContext { this.paths = new ArrayList<>(); this.sourceOperators = new ArrayList<>(); this.deviceIDToContext = null; - this.externalTsFilePaths = null; } public void setQueryDataSourceType(QueryDataSourceType queryDataSourceType) { @@ -74,19 +71,6 @@ public class DataDriverContext extends DriverContext { deviceIDToContext = null; } - public void setExternalTsFilePaths(List<String> externalTsFilePaths) { - this.externalTsFilePaths = externalTsFilePaths; - } - - public List<String> getExternalTsFilePaths() { - return externalTsFilePaths; - } - - public void clearExternalTsFilePaths() { - // friendly for gc - externalTsFilePaths = null; - } - public void addPath(IFullPath path) { this.paths.add(path); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 62aa75f85c0..56214b971f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -618,8 +618,12 @@ public class FragmentInstanceContext extends QueryContext { this.devicePathsToContext = devicePathsToContext; } - public void setExternalTsFilePaths(List<String> externalTsFilePaths) { - this.externalTsFilePaths = externalTsFilePaths; + public void addExternalTsFilePaths(List<String> externalTsFilePaths) { + if (this.externalTsFilePaths == null) { + this.externalTsFilePaths = new ArrayList<>(externalTsFilePaths); + return; + } + this.externalTsFilePaths.addAll(externalTsFilePaths); } public MemoryReservationManager getMemoryReservationContext() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MultiTsFileResourceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MultiTsFileResourceIterator.java index 67bddc4facb..c55d336a224 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MultiTsFileResourceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MultiTsFileResourceIterator.java @@ -151,9 +151,8 @@ public class MultiTsFileResourceIterator { } private boolean isDeviceMatched(IDeviceID deviceID) { - return tableName.equalsIgnoreCase(deviceID.getTableName()) - && (deviceFilter == null - || Boolean.TRUE.equals(deviceFilter.accept(deviceFilterVisitor, deviceID))); + return deviceFilter == null + || Boolean.TRUE.equals(deviceFilter.accept(deviceFilterVisitor, deviceID)); } private class TsFileResourceDeviceIterator { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/OrderedExternalTsFileTableScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/OrderedExternalTsFileTableScanOperator.java index 1476ed889aa..3b84e1a1908 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/OrderedExternalTsFileTableScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/OrderedExternalTsFileTableScanOperator.java @@ -36,10 +36,9 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.read.TsFileDeviceIterator; +import org.apache.tsfile.read.LazyTsFileDeviceIterator; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; import java.io.IOException; @@ -136,17 +135,20 @@ public class OrderedExternalTsFileTableScanOperator extends AbstractTableScanOpe private void collectDeviceInfos( TsFileResource resource, List<ExternalTsFileDeviceInfo> deviceInfos) { try (TsFileSequenceReader reader = new TsFileSequenceReader(resource.getTsFilePath())) { - TsFileDeviceIterator deviceIterator = - reader.getTableDevicesIteratorWithIsAligned(tableName, contextValue -> {}); + LazyTsFileDeviceIterator deviceIterator = + new LazyTsFileDeviceIterator( + reader, + tableName, + ((OperatorContext) operatorContext) + .getInstanceContext() + .getQueryStatistics() + .getLoadTimeSeriesMetadataActualIOSize() + ::addAndGet); while (deviceIterator.hasNext()) { - Pair<IDeviceID, Boolean> deviceInfo = deviceIterator.next(); - IDeviceID deviceID = deviceInfo.left; + IDeviceID deviceID = deviceIterator.next(); if (!isDeviceMatched(deviceID)) { continue; } - if (!isDeviceFilterMatched(deviceID)) { - continue; - } deviceInfos.add( new ExternalTsFileDeviceInfo( deviceID, resource, deviceIterator.getCurrentDeviceMeasurementNodeOffset())); @@ -167,10 +169,6 @@ public class OrderedExternalTsFileTableScanOperator extends AbstractTableScanOpe } private boolean isDeviceMatched(IDeviceID deviceID) { - return tableName.equalsIgnoreCase(deviceID.getTableName()); - } - - private boolean isDeviceFilterMatched(IDeviceID deviceID) { return deviceFilter == null || Boolean.TRUE.equals(deviceFilter.accept(deviceFilterVisitor, deviceID)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java index 3f678f7abdd..a857e063304 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/DataNodeTableOperatorGenerator.java @@ -1149,9 +1149,9 @@ public class DataNodeTableOperatorGenerator DataDriverContext dataDriverContext = (DataDriverContext) context.getDriverContext(); dataDriverContext.addSourceOperator(externalTsFileTableScanOperator); - dataDriverContext.setExternalTsFilePaths(node.getTsFilePaths()); dataDriverContext.setQueryDataSourceType(QueryDataSourceType.EXTERNAL_TSFILE_SCAN); dataDriverContext.setInputDriver(true); + context.getInstanceContext().addExternalTsFilePaths(node.getTsFilePaths()); return externalTsFileTableScanOperator; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java index f05c49b466d..95f28052c1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java @@ -124,7 +124,6 @@ public class LocalExecutionPlanner { instanceContext.setSourcePaths(collectSourcePaths(context)); instanceContext.setDevicePathsToContext(collectDevicePathsToContext(context)); - instanceContext.setExternalTsFilePaths(collectExternalTsFilePaths(context)); instanceContext.setQueryDataSourceType( getQueryDataSourceType((DataDriverContext) context.getDriverContext())); @@ -265,21 +264,6 @@ public class LocalExecutionPlanner { return sourcePaths; } - private List<String> collectExternalTsFilePaths(LocalExecutionPlanContext context) { - List<String> externalTsFilePaths = new ArrayList<>(); - context - .getPipelineDriverFactories() - .forEach( - pipeline -> { - DataDriverContext dataDriverContext = (DataDriverContext) pipeline.getDriverContext(); - if (dataDriverContext.getExternalTsFilePaths() != null) { - externalTsFilePaths.addAll(dataDriverContext.getExternalTsFilePaths()); - } - dataDriverContext.clearExternalTsFilePaths(); - }); - return externalTsFilePaths; - } - public synchronized boolean forceAllocateFreeMemoryForOperators(long memoryInBytes) { // TODO @spricoder: consider a better way if (OPERATORS_MEMORY_BLOCK.getFreeMemoryInBytes() - memoryInBytes diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java index e38ce40d48b..4338fe8ad43 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java @@ -97,6 +97,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CteScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExternalTsFileScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableDiskUsageInformationSchemaTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; @@ -661,6 +662,10 @@ public class PlanGraphPrinter implements PlanVisitor<List<String>, PlanGraphPrin if (node instanceof DeviceTableScanNode) { deviceTableScanNode = (DeviceTableScanNode) node; } + ExternalTsFileScanNode externalTsFileScanNode = null; + if (node instanceof ExternalTsFileScanNode) { + externalTsFileScanNode = (ExternalTsFileScanNode) node; + } List<String> boxValue = new ArrayList<>(); boxValue.add(node.toString()); @@ -676,6 +681,22 @@ public class PlanGraphPrinter implements PlanVisitor<List<String>, PlanGraphPrin String.format("TimePredicate: %s", deviceTableScanNode.getTimePredicate().get())); } } + if (externalTsFileScanNode != null) { + boxValue.add(String.format("ScanOrder: %s", externalTsFileScanNode.getScanOrder())); + boxValue.add(String.format("TsFilePaths: %s", externalTsFileScanNode.getTsFilePaths())); + externalTsFileScanNode + .getPushedOrderingScheme() + .ifPresent( + orderingScheme -> + boxValue.add(String.format("PushedOrderingScheme: %s", orderingScheme))); + externalTsFileScanNode + .getTimePredicate() + .ifPresent( + timePredicate -> boxValue.add(String.format("TimePredicate: %s", timePredicate))); + externalTsFileScanNode + .getTagPredicate() + .ifPresent(tagPredicate -> boxValue.add(String.format("TagPredicate: %s", tagPredicate))); + } if (node.getPushDownPredicate() != null) { boxValue.add(String.format("PushDownPredicate: %s", node.getPushDownPredicate())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java index 510c748acee..7bcf09fd73a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/schema/ConvertSchemaPredicateToFilterVisitor.java @@ -55,6 +55,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.Predic import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -234,8 +235,24 @@ public class ConvertSchemaPredicateToFilterVisitor } @Override - public SchemaFilter visitBetweenPredicate(final BetweenPredicate node, final Context context) { - return visitExpression(node, context); + public @Nullable SchemaFilter visitBetweenPredicate( + final BetweenPredicate node, final Context context) { + final SchemaFilter lowerBoundFilter = + new ComparisonExpression( + ComparisonExpression.Operator.LESS_THAN_OR_EQUAL, + node.getMin(), + node.getValue()) + .accept(this, context); + final SchemaFilter upperBoundFilter = + new ComparisonExpression( + ComparisonExpression.Operator.LESS_THAN_OR_EQUAL, + node.getValue(), + node.getMax()) + .accept(this, context); + if (Objects.isNull(lowerBoundFilter) || Objects.isNull(upperBoundFilter)) { + return null; + } + return new AndFilter(Arrays.asList(lowerBoundFilter, upperBoundFilter)); } private SchemaFilter wrapTagOrAttributeFilter(
