This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit caf7790797c3367069a022e8af180bdaa066cb50 Author: Gabriel <[email protected]> AuthorDate: Fri Jan 19 16:12:35 2024 +0800 [pipelineX](filescan) Support parallel executing for external table scanning (#30121) --- .../src/main/java/org/apache/doris/planner/OlapScanNode.java | 5 +++++ .../src/main/java/org/apache/doris/planner/ScanNode.java | 7 +++++-- .../org/apache/doris/planner/external/FileQueryScanNode.java | 12 ++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index e5381a2018f..be2c9a24c91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1736,6 +1736,11 @@ public class OlapScanNode extends ScanNode { return true; } + + @Override + public int getScanRangeNum() { + return getScanTabletIds().size(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index 7ddf2ae1d4c..9881fe6f6e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -725,8 +725,11 @@ public abstract class ScanNode extends PlanNode { && context.getSessionVariable().isIgnoreStorageDataDistribution() && context.getSessionVariable().getEnablePipelineXEngine() && !fragment.isHasNullAwareLeftAntiJoin() - && ((this instanceof OlapScanNode) && ((OlapScanNode) this).getScanTabletIds().size() - < ConnectContext.get().getSessionVariable().getParallelExecInstanceNum()); + && getScanRangeNum() < ConnectContext.get().getSessionVariable().getParallelExecInstanceNum(); + } + + public int getScanRangeNum() { + return Integer.MAX_VALUE; } public boolean haveLimitAndConjunts() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java index 71c731498a6..4a3c3d2ff9a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java @@ -485,6 +485,18 @@ public abstract class FileQueryScanNode extends FileScanNode { params.setSlotNameToSchemaPos(columnNameToPosition); } + @Override + public int getScanRangeNum() { + Preconditions.checkNotNull(scanRangeLocations); + int i = 0; + for (TScanRangeLocations tScanRangeLocations : scanRangeLocations) { + TScanRange tScanRange = tScanRangeLocations.getScanRange(); + TFileScanRange tFileScanRange = tScanRange.getExtScanRange().getFileScanRange(); + i += tFileScanRange.getRangesSize(); + } + return i; + } + protected abstract TFileType getLocationType() throws UserException; protected abstract TFileType getLocationType(String location) throws UserException; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
