This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 3075749727d [Improvement](nereids)Support ODBC table for new planner.
#29129 (#29130)
3075749727d is described below
commit 3075749727d0ee2ffe04153460e2b7bddb1b4ded
Author: zy-kkk <[email protected]>
AuthorDate: Thu Dec 28 10:24:35 2023 +0800
[Improvement](nereids)Support ODBC table for new planner. #29129 (#29130)
---
.../org/apache/doris/nereids/cost/CostModelV1.java | 7 ++
.../org/apache/doris/nereids/cost/CostModelV2.java | 6 ++
.../glue/translator/PhysicalPlanTranslator.java | 26 ++++++
.../doris/nereids/jobs/executor/Rewriter.java | 2 +
.../properties/ChildOutputPropertyDeriver.java | 6 ++
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +
.../org/apache/doris/nereids/rules/RuleType.java | 2 +
.../doris/nereids/rules/analysis/BindRelation.java | 3 +
.../LogicalOdbcScanToPhysicalOdbcScan.java | 42 +++++++++
.../rules/rewrite/PushConjunctsIntoOdbcScan.java | 39 ++++++++
.../doris/nereids/stats/StatsCalculator.java | 13 +++
.../trees/copier/LogicalPlanDeepCopier.java | 13 +++
.../apache/doris/nereids/trees/plans/PlanType.java | 2 +
.../trees/plans/logical/LogicalOdbcScan.java | 104 +++++++++++++++++++++
.../trees/plans/physical/PhysicalOdbcScan.java | 102 ++++++++++++++++++++
.../trees/plans/visitor/RelationVisitor.java | 10 ++
.../doris/planner/external/odbc/OdbcScanNode.java | 44 +++++++--
17 files changed, 417 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
index e50f546ea4d..9d43b3c3195 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
@@ -34,6 +34,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
@@ -138,6 +139,12 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
return CostV1.ofCpu(statistics.getRowCount());
}
+ @Override
+ public Cost visitPhysicalOdbcScan(PhysicalOdbcScan physicalOdbcScan,
PlanContext context) {
+ Statistics statistics = context.getStatisticsWithCheck();
+ return CostV1.ofCpu(statistics.getRowCount());
+ }
+
@Override
public Cost visitPhysicalEsScan(PhysicalEsScan physicalEsScan, PlanContext
context) {
Statistics statistics = context.getStatisticsWithCheck();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
index c90f28c6b8f..4439409948f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
@@ -34,6 +34,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
@@ -135,6 +136,11 @@ class CostModelV2 extends PlanVisitor<Cost, PlanContext> {
return calculateScanWithoutRF(context.getStatisticsWithCheck());
}
+ @Override
+ public Cost visitPhysicalOdbcScan(PhysicalOdbcScan physicalOdbcScan,
PlanContext context) {
+ return calculateScanWithoutRF(context.getStatisticsWithCheck());
+ }
+
@Override
public Cost visitPhysicalEsScan(PhysicalEsScan physicalEsScan, PlanContext
context) {
return calculateScanWithoutRF(context.getStatisticsWithCheck());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 7036652a028..0f5b5dd597d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -41,6 +41,7 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Function.NullableMode;
+import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
@@ -110,6 +111,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
@@ -173,6 +175,7 @@ import org.apache.doris.planner.external.MaxComputeScanNode;
import org.apache.doris.planner.external.hudi.HudiScanNode;
import org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
+import org.apache.doris.planner.external.odbc.OdbcScanNode;
import org.apache.doris.planner.external.paimon.PaimonScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
@@ -582,6 +585,29 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
return planFragment;
}
+ @Override
+ public PlanFragment visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan,
PlanTranslatorContext context) {
+ List<Slot> slots = odbcScan.getOutput();
+ TableIf table = odbcScan.getTable();
+ TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table,
context);
+ OdbcScanNode odbcScanNode = new
OdbcScanNode(odbcScan.translatePlanNodeId(), tupleDescriptor,
+ (OdbcTable) table);
+
odbcScanNode.addConjuncts(translateToLegacyConjuncts(odbcScan.getConjuncts()));
+ Utils.execWithUncheckedException(odbcScanNode::init);
+ context.addScanNode(odbcScanNode);
+ context.getRuntimeTranslator().ifPresent(
+ runtimeFilterGenerator ->
runtimeFilterGenerator.getTargetOnScanNode(odbcScan.getRelationId()).forEach(
+ expr ->
runtimeFilterGenerator.translateRuntimeFilterTarget(expr, odbcScanNode, context)
+ )
+ );
+ Utils.execWithUncheckedException(odbcScanNode::finalizeForNereids);
+ DataPartition dataPartition = DataPartition.RANDOM;
+ PlanFragment planFragment = new PlanFragment(context.nextFragmentId(),
odbcScanNode, dataPartition);
+ context.addPlanFragment(planFragment);
+ updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), odbcScan);
+ return planFragment;
+ }
+
@Override
public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan,
PlanTranslatorContext context) {
List<Slot> slots = olapScan.getOutput();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 24495ccc620..2f789babd8a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -87,6 +87,7 @@ import org.apache.doris.nereids.rules.rewrite.PullUpCteAnchor;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderApply;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan;
+import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoOdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin;
import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation;
import org.apache.doris.nereids.rules.rewrite.PushProjectThroughUnion;
@@ -286,6 +287,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
new PruneEmptyPartition(),
new PruneFileScanPartition(),
new PushConjunctsIntoJdbcScan(),
+ new PushConjunctsIntoOdbcScan(),
new PushConjunctsIntoEsScan()
)
),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
index c1591f9d78c..4bcf6cafb84 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
@@ -47,6 +47,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
@@ -148,6 +149,11 @@ public class ChildOutputPropertyDeriver extends
PlanVisitor<PhysicalProperties,
return PhysicalProperties.STORAGE_ANY;
}
+ @Override
+ public PhysicalProperties visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan,
PlanContext context) {
+ return PhysicalProperties.STORAGE_ANY;
+ }
+
@Override
public PhysicalProperties visitPhysicalOlapScan(PhysicalOlapScan olapScan,
PlanContext context) {
return new PhysicalProperties(olapScan.getDistributionSpec());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index b7a43d66984..05846fbd123 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -59,6 +59,7 @@ import
org.apache.doris.nereids.rules.implementation.LogicalJdbcScanToPhysicalJd
import org.apache.doris.nereids.rules.implementation.LogicalJoinToHashJoin;
import
org.apache.doris.nereids.rules.implementation.LogicalJoinToNestedLoopJoin;
import
org.apache.doris.nereids.rules.implementation.LogicalLimitToPhysicalLimit;
+import
org.apache.doris.nereids.rules.implementation.LogicalOdbcScanToPhysicalOdbcScan;
import
org.apache.doris.nereids.rules.implementation.LogicalOlapScanToPhysicalOlapScan;
import
org.apache.doris.nereids.rules.implementation.LogicalOlapTableSinkToPhysicalOlapTableSink;
import
org.apache.doris.nereids.rules.implementation.LogicalOneRowRelationToPhysicalOneRowRelation;
@@ -155,6 +156,7 @@ public class RuleSet {
.add(new LogicalSchemaScanToPhysicalSchemaScan())
.add(new LogicalFileScanToPhysicalFileScan())
.add(new LogicalJdbcScanToPhysicalJdbcScan())
+ .add(new LogicalOdbcScanToPhysicalOdbcScan())
.add(new LogicalEsScanToPhysicalEsScan())
.add(new LogicalProjectToPhysicalProject())
.add(new LogicalLimitToPhysicalLimit())
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index ec8d53de8f0..ba2e5a6456c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -223,6 +223,7 @@ public enum RuleType {
OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
FILE_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
PUSH_CONJUNCTS_INTO_JDBC_SCAN(RuleTypeClass.REWRITE),
+ PUSH_CONJUNCTS_INTO_ODBC_SCAN(RuleTypeClass.REWRITE),
PUSH_CONJUNCTS_INTO_ES_SCAN(RuleTypeClass.REWRITE),
OLAP_SCAN_TABLET_PRUNE(RuleTypeClass.REWRITE),
PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE),
@@ -332,6 +333,7 @@ public enum RuleType {
LOGICAL_SCHEMA_SCAN_TO_PHYSICAL_SCHEMA_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_FILE_SCAN_TO_PHYSICAL_FILE_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_JDBC_SCAN_TO_PHYSICAL_JDBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
+ LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_ES_SCAN_TO_PHYSICAL_ES_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_OLAP_TABLE_SINK_TO_PHYSICAL_OLAP_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
LOGICAL_RESULT_SINK_TO_PHYSICAL_RESULT_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 6980817db09..60f46ae7b03 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -51,6 +51,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalEsScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
@@ -227,6 +228,8 @@ public class BindRelation extends OneAnalysisRuleFactory {
case JDBC_EXTERNAL_TABLE:
case JDBC:
return new LogicalJdbcScan(unboundRelation.getRelationId(),
table, tableQualifier);
+ case ODBC:
+ return new LogicalOdbcScan(unboundRelation.getRelationId(),
table, tableQualifier);
case ES_EXTERNAL_TABLE:
return new LogicalEsScan(unboundRelation.getRelationId(),
(EsExternalTable) table, tableQualifier);
default:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java
new file mode 100644
index 00000000000..22aea72f736
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.implementation;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
+
+import java.util.Optional;
+
+/**
+ * Implementation rule that convert logical OdbcScan to physical OdbcScan.
+ */
+public class LogicalOdbcScanToPhysicalOdbcScan extends
OneImplementationRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalOdbcScan().then(odbcScan ->
+ new PhysicalOdbcScan(
+ odbcScan.getRelationId(),
+ odbcScan.getTable(),
+ odbcScan.getQualifier(),
+ Optional.empty(),
+ odbcScan.getLogicalProperties(),
+ odbcScan.getConjuncts())
+ ).toRule(RuleType.LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java
new file mode 100644
index 00000000000..8a1ba4d1b67
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
+
+/**
+ * Rewrite odbc plan to set the conjuncts.
+ */
+public class PushConjunctsIntoOdbcScan extends OneRewriteRuleFactory {
+
+ @Override
+ public Rule build() {
+ return logicalFilter(logicalOdbcScan()).thenApply(ctx -> {
+ LogicalFilter<LogicalOdbcScan> filter = ctx.root;
+ LogicalOdbcScan scan = filter.child();
+ LogicalOdbcScan rewrittenScan =
scan.withConjuncts(filter.getConjuncts());
+ return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan);
+ }).toRule(RuleType.PUSH_CONJUNCTS_INTO_ODBC_SCAN);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index ba7d551a4ef..64f85c47a80 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -64,6 +64,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
@@ -95,6 +96,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
@@ -322,6 +324,12 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
return computeCatalogRelation(jdbcScan);
}
+ @Override
+ public Statistics visitLogicalOdbcScan(LogicalOdbcScan odbcScan, Void
context) {
+ odbcScan.getExpressions();
+ return computeCatalogRelation(odbcScan);
+ }
+
@Override
public Statistics visitLogicalEsScan(LogicalEsScan esScan, Void context) {
esScan.getExpressions();
@@ -464,6 +472,11 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
return computeCatalogRelation(jdbcScan);
}
+ @Override
+ public Statistics visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, Void
context) {
+ return computeCatalogRelation(odbcScan);
+ }
+
@Override
public Statistics visitPhysicalEsScan(PhysicalEsScan esScan, Void context)
{
return computeCatalogRelation(esScan);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
index 45aa14f6e46..0d951158e2e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
@@ -49,6 +49,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
@@ -238,6 +239,18 @@ public class LogicalPlanDeepCopier extends
DefaultPlanRewriter<DeepCopierContext
return newJdbcScan;
}
+ @Override
+ public Plan visitLogicalOdbcScan(LogicalOdbcScan odbcScan,
DeepCopierContext context) {
+ if
(context.getRelationReplaceMap().containsKey(odbcScan.getRelationId())) {
+ return
context.getRelationReplaceMap().get(odbcScan.getRelationId());
+ }
+ LogicalOdbcScan newOdbcScan = new
LogicalOdbcScan(StatementScopeIdGenerator.newRelationId(),
+ odbcScan.getTable(), odbcScan.getQualifier());
+ updateReplaceMapWithOutput(odbcScan, newOdbcScan,
context.exprIdReplaceMap);
+ context.putRelation(odbcScan.getRelationId(), newOdbcScan);
+ return newOdbcScan;
+ }
+
@Override
public Plan visitLogicalEsScan(LogicalEsScan esScan, DeepCopierContext
context) {
if
(context.getRelationReplaceMap().containsKey(esScan.getRelationId())) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index da98e0e4d14..83daeec651e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -33,6 +33,7 @@ public enum PlanType {
LOGICAL_EMPTY_RELATION,
LOGICAL_ES_SCAN,
LOGICAL_JDBC_SCAN,
+ LOGICAL_ODBC_SCAN,
LOGICAL_OLAP_SCAN,
LOGICAL_ONE_ROW_RELATION,
LOGICAL_SCHEMA_SCAN,
@@ -82,6 +83,7 @@ public enum PlanType {
PHYSICAL_ES_SCAN,
PHYSICAL_FILE_SCAN,
PHYSICAL_JDBC_SCAN,
+ PHYSICAL_ODBC_SCAN,
PHYSICAL_ONE_ROW_RELATION,
PHYSICAL_OLAP_SCAN,
PHYSICAL_SCHEMA_SCAN,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java
new file mode 100644
index 00000000000..840cb1113a4
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.catalog.OdbcTable;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Logical scan for external odbc table.
+ */
+public class LogicalOdbcScan extends LogicalCatalogRelation {
+
+ private final Set<Expression> conjuncts;
+
+ public LogicalOdbcScan(RelationId id, TableIf table, List<String>
qualifier,
+ Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties,
+ Set<Expression> conjuncts) {
+ super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier,
+ groupExpression, logicalProperties);
+ this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts,
"conjuncts should not be null"));
+ }
+
+ public LogicalOdbcScan(RelationId id, TableIf table, List<String>
qualifier) {
+ this(id, table, qualifier, Optional.empty(), Optional.empty(),
ImmutableSet.of());
+ }
+
+ @Override
+ public TableIf getTable() {
+ Preconditions.checkArgument(table instanceof OdbcTable,
+ String.format("Table %s is not OdbcTable", table.getName()));
+ return table;
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("LogicalOdbcScan",
+ "qualified", qualifiedName(),
+ "output", getOutput()
+ );
+ }
+
+ @Override
+ public LogicalOdbcScan withGroupExpression(Optional<GroupExpression>
groupExpression) {
+ return new LogicalOdbcScan(relationId, table, qualifier,
groupExpression,
+ Optional.of(getLogicalProperties()), conjuncts);
+ }
+
+ public LogicalOdbcScan withConjuncts(Set<Expression> conjuncts) {
+ return new LogicalOdbcScan(relationId, table, qualifier,
groupExpression,
+ Optional.of(getLogicalProperties()), conjuncts);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression>
groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan>
children) {
+ return new LogicalOdbcScan(relationId, table, qualifier,
groupExpression, logicalProperties, conjuncts);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitLogicalOdbcScan(this, context);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return super.equals(o) && Objects.equals(conjuncts, ((LogicalOdbcScan)
o).conjuncts);
+ }
+
+ public Set<Expression> getConjuncts() {
+ return this.conjuncts;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java
new file mode 100644
index 00000000000..f63b5a8d420
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.statistics.Statistics;
+
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Physical odbc scan for odbc table.
+ */
+public class PhysicalOdbcScan extends PhysicalCatalogRelation {
+
+ private final Set<Expression> conjuncts;
+
+ /**
+ * Constructor for PhysicalOdbcScan.
+ */
+ public PhysicalOdbcScan(RelationId id, TableIf table, List<String>
qualifier,
+ Optional<GroupExpression> groupExpression, LogicalProperties
logicalProperties, Set<Expression> conjuncts) {
+ this(id, table, qualifier, groupExpression, logicalProperties,
+ null, null, conjuncts);
+ }
+
+ /**
+ * Constructor for PhysicalOdbcScan.
+ */
+ public PhysicalOdbcScan(RelationId id, TableIf table, List<String>
qualifier,
+ Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties, PhysicalProperties
physicalProperties, Statistics statistics,
+ Set<Expression> conjuncts) {
+ super(id, PlanType.PHYSICAL_ODBC_SCAN, table, qualifier,
groupExpression,
+ logicalProperties, physicalProperties, statistics);
+ this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts,
"conjuncts should not be null"));
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toSqlString("PhysicalOdbcScan",
+ "qualified", Utils.qualifiedName(qualifier, table.getName()),
+ "output", getOutput(),
+ "stats", statistics
+ );
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitPhysicalOdbcScan(this, context);
+ }
+
+ @Override
+ public PhysicalOdbcScan withGroupExpression(Optional<GroupExpression>
groupExpression) {
+ return new PhysicalOdbcScan(relationId, table, qualifier,
groupExpression, getLogicalProperties(), conjuncts);
+ }
+
+ @Override
+ public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression>
groupExpression,
+ Optional<LogicalProperties> logicalProperties, List<Plan>
children) {
+ return new PhysicalOdbcScan(relationId, table, qualifier,
groupExpression, logicalProperties.get(), conjuncts);
+ }
+
+ @Override
+ public PhysicalOdbcScan withPhysicalPropertiesAndStats(PhysicalProperties
physicalProperties,
+ Statistics
statistics) {
+ return new PhysicalOdbcScan(relationId, table, qualifier,
groupExpression,
+ getLogicalProperties(), physicalProperties, statistics,
conjuncts);
+ }
+
+ public Set<Expression> getConjuncts() {
+ return this.conjuncts;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
index 65a03505021..adfe4991665 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
@@ -25,6 +25,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
@@ -35,6 +36,7 @@ import
org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
@@ -90,6 +92,10 @@ public interface RelationVisitor<R, C> {
return visitLogicalRelation(jdbcScan, context);
}
+ default R visitLogicalOdbcScan(LogicalOdbcScan odbcScan, C context) {
+ return visitLogicalRelation(odbcScan, context);
+ }
+
default R visitLogicalOlapScan(LogicalOlapScan olapScan, C context) {
return visitLogicalRelation(olapScan, context);
}
@@ -131,6 +137,10 @@ public interface RelationVisitor<R, C> {
return visitPhysicalRelation(jdbcScan, context);
}
+ default R visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, C context) {
+ return visitPhysicalRelation(odbcScan, context);
+ }
+
default R visitPhysicalOlapScan(PhysicalOlapScan olapScan, C context) {
return visitPhysicalRelation(olapScan, context);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
index bf4e835e4f1..d90140438f1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.ExprSubstitutionMap;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.SlotDescriptor;
+import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Column;
@@ -31,6 +32,7 @@ import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.planner.external.ExternalScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
@@ -52,6 +54,7 @@ import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
/**
* Full scan of an ODBC table.
@@ -78,6 +81,22 @@ public class OdbcScanNode extends ExternalScanNode {
this.tbl = tbl;
}
+ @Override
+ public void init(Analyzer analyzer) throws UserException {
+ super.init(analyzer);
+ }
+
+ /**
+ * Used for Nereids. Should NOT use this function in anywhere else.
+ */
+ @Override
+ public void init() throws UserException {
+ super.init();
+ numNodes = numNodes <= 0 ? 1 : numNodes;
+
StatsRecursiveDerive.getStatsRecursiveDerive().statsRecursiveDerive(this);
+ cardinality = (long) statsDeriveResult.getRowCount();
+ }
+
@Override
protected String debugString() {
MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this);
@@ -87,11 +106,24 @@ public class OdbcScanNode extends ExternalScanNode {
@Override
public void finalize(Analyzer analyzer) throws UserException {
// Convert predicates to Odbc columns and filters.
- createOdbcColumns(analyzer);
- createOdbcFilters(analyzer);
+ createOdbcColumns();
+ createOdbcFilters();
+ createScanRangeLocations();
+ }
+
+ @Override
+ public void finalizeForNereids() throws UserException {
+ createOdbcColumns();
+ createOdbcFilters();
createScanRangeLocations();
}
+ @Override
+ public void updateRequiredSlots(PlanTranslatorContext context, Set<SlotId>
requiredByProjectSlotIdSet)
+ throws UserException {
+ createOdbcColumns();
+ }
+
@Override
protected void createScanRangeLocations() throws UserException {
scanRangeLocations =
Lists.newArrayList(createSingleScanRangeLocations(backendPolicy));
@@ -152,7 +184,8 @@ public class OdbcScanNode extends ExternalScanNode {
return sql.toString();
}
- private void createOdbcColumns(Analyzer analyzer) {
+ private void createOdbcColumns() {
+ columns.clear();
for (SlotDescriptor slot : desc.getSlots()) {
if (!slot.isMaterialized()) {
continue;
@@ -161,16 +194,15 @@ public class OdbcScanNode extends ExternalScanNode {
columns.add(JdbcTable.databaseProperName(odbcType, col.getName()));
}
// this happens when count(*)
- if (0 == columns.size()) {
+ if (columns.isEmpty()) {
columns.add("*");
}
}
// We convert predicates of the form <slotref> op <constant> to Odbc
filters
- private void createOdbcFilters(Analyzer analyzer) {
+ private void createOdbcFilters() {
if (conjuncts.isEmpty()) {
return;
-
}
List<SlotRef> slotRefs = Lists.newArrayList();
Expr.collectList(conjuncts, SlotRef.class, slotRefs);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
