This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new a4e793752fc [bugfix](iceberg)revert count(*) directly returned by fe
for 2.1 (#38566) (#38655)
a4e793752fc is described below
commit a4e793752fccef8e49d98d6992df3ed352eb0942
Author: wuwenchi <[email protected]>
AuthorDate: Thu Aug 1 18:56:19 2024 +0800
[bugfix](iceberg)revert count(*) directly returned by fe for 2.1 (#38566)
(#38655)
bp: #38566
---
.../org/apache/doris/datasource/FileScanNode.java | 1 -
.../datasource/iceberg/source/IcebergScanNode.java | 13 ++++++------
.../org/apache/doris/nereids/NereidsPlanner.java | 24 +---------------------
.../org/apache/doris/planner/OriginalPlanner.java | 16 ---------------
4 files changed, 8 insertions(+), 46 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
index d202e01f524..f08fb0a503f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
@@ -71,7 +71,6 @@ public abstract class FileScanNode extends ExternalScanNode {
protected long totalPartitionNum = 0;
protected long readPartitionNum = 0;
protected long fileSplitSize;
- public long rowCount = 0;
public FileScanNode(PlanNodeId id, TupleDescriptor desc, String
planNodeName, StatisticalType statisticalType,
boolean needCheckColumnPriv) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index bb200fdf027..38d9c8e8215 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -75,6 +75,7 @@ import org.apache.iceberg.util.TableScanUtil;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -209,12 +210,6 @@ public class IcebergScanNode extends FileQueryScanNode {
HashSet<String> partitionPathSet = new HashSet<>();
boolean isPartitionedTable = icebergTable.spec().isPartitioned();
- long rowCount = getCountFromSnapshot();
- if (getPushDownAggNoGroupingOp().equals(TPushAggOp.COUNT) && rowCount
>= 0) {
- this.rowCount = rowCount;
- return new ArrayList<>();
- }
-
CloseableIterable<FileScanTask> fileScanTasks =
TableScanUtil.splitFiles(scan.planFiles(), splitSize);
try (CloseableIterable<CombinedScanTask> combinedScanTasks =
TableScanUtil.planTasks(fileScanTasks, splitSize, 1, 0)) {
@@ -268,6 +263,12 @@ public class IcebergScanNode extends FileQueryScanNode {
throw new UserException(e.getMessage(), e.getCause());
}
+ TPushAggOp aggOp = getPushDownAggNoGroupingOp();
+ if (aggOp.equals(TPushAggOp.COUNT) && getCountFromSnapshot() >= 0) {
+ // we can create a special empty split and skip the plan process
+ return splits.isEmpty() ? splits :
Collections.singletonList(splits.get(0));
+ }
+
readPartitionNum = partitionPathSet.size();
return splits;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 12bb8c3623c..67f4bb73b18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -26,7 +26,6 @@ import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.NereidsException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.profile.SummaryProfile;
-import org.apache.doris.datasource.iceberg.source.IcebergScanNode;
import org.apache.doris.mysql.FieldInfo;
import org.apache.doris.nereids.CascadesContext.Lock;
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -54,18 +53,14 @@ import org.apache.doris.nereids.trees.plans.Plan;
import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache;
-import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
-import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSqlCache;
import org.apache.doris.planner.PlanFragment;
import org.apache.doris.planner.Planner;
import org.apache.doris.planner.RuntimeFilter;
import org.apache.doris.planner.ScanNode;
-import org.apache.doris.qe.CommonResultSet;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ResultSet;
-import org.apache.doris.qe.ResultSetMetaData;
import org.apache.doris.qe.SessionVariable;
import com.google.common.annotations.VisibleForTesting;
@@ -75,7 +70,6 @@ import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
@@ -572,23 +566,7 @@ public class NereidsPlanner extends Planner {
}
}
- if (physicalPlan instanceof PhysicalResultSink
- && physicalPlan.child(0) instanceof PhysicalHashAggregate &&
!getScanNodes().isEmpty()
- && getScanNodes().get(0) instanceof IcebergScanNode) {
- List<Column> columns = Lists.newArrayList();
- NamedExpression output = physicalPlan.getOutput().get(0);
- columns.add(new Column(output.getName(),
output.getDataType().toCatalogDataType()));
- if (((IcebergScanNode) getScanNodes().get(0)).rowCount > 0) {
- ResultSetMetaData metadata = new
CommonResultSet.CommonResultSetMetaData(columns);
- ResultSet resultSet = new CommonResultSet(metadata,
Collections.singletonList(
- Lists.newArrayList(String.valueOf(((IcebergScanNode)
getScanNodes().get(0)).rowCount))));
- // only support one iceberg scan node and one count, e.g.
select count(*) from icetbl;
- return Optional.of(resultSet);
- }
- return Optional.empty();
- } else {
- return Optional.empty();
- }
+ return Optional.empty();
}
private void setFormatOptions() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
index 70e442546c3..2657232db89 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
@@ -41,7 +41,6 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.UserException;
-import org.apache.doris.datasource.iceberg.source.IcebergScanNode;
import org.apache.doris.nereids.PlannerHook;
import org.apache.doris.qe.CommonResultSet;
import org.apache.doris.qe.ConnectContext;
@@ -642,21 +641,6 @@ public class OriginalPlanner extends Planner {
List<Column> columns = new ArrayList<>(selectItems.size());
List<String> columnLabels = parsedSelectStmt.getColLabels();
List<String> data = new ArrayList<>();
- if ((singleNodePlanner.getScanNodes().size() > 0 &&
singleNodePlanner.getScanNodes().get(0)
- instanceof IcebergScanNode) && (((IcebergScanNode)
getScanNodes().get(0)).rowCount > 0)) {
- SelectListItem item = selectItems.get(0);
- Expr expr = item.getExpr();
- String columnName = columnLabels.get(0);
- columns.add(new Column(columnName, expr.getType()));
- data.add(String.valueOf(((IcebergScanNode)
getScanNodes().get(0)).rowCount));
- ResultSetMetaData metadata = new
CommonResultSet.CommonResultSetMetaData(columns);
- ResultSet resultSet = new CommonResultSet(metadata,
Collections.singletonList(data));
- // only support one iceberg scan node and one count, e.g. select
count(*) from icetbl;
- return Optional.of(resultSet);
- }
- if (!parsedSelectStmt.getTableRefs().isEmpty()) {
- return Optional.empty();
- }
FormatOptions options = FormatOptions.getDefault();
for (int i = 0; i < selectItems.size(); i++) {
SelectListItem item = selectItems.get(i);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
