This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 967801ca666 [feature](Nereids) support query on ES table in internal
catalog (#40449) (#40754)
967801ca666 is described below
commit 967801ca666f84a60010bef1f6ca5ca777d5b901
Author: morrySnow <[email protected]>
AuthorDate: Thu Sep 12 20:01:48 2024 +0800
[feature](Nereids) support query on ES table in internal catalog (#40449)
(#40754)
pick from master #40449
---
.../glue/translator/PhysicalPlanTranslator.java | 6 +-
.../doris/nereids/rules/analysis/BindRelation.java | 11 +---
.../nereids/trees/plans/logical/LogicalEsScan.java | 21 +++----
.../trees/plans/physical/PhysicalEsScan.java | 11 +---
.../apache/doris/planner/StatisticDeriveTest.java | 27 ---------
.../test_push_conjunct_external_table.groovy | 67 ----------------------
6 files changed, 16 insertions(+), 127 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 129854b7ff9..78a6a083090 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -48,6 +48,7 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.FileQueryScanNode;
+import org.apache.doris.datasource.es.EsExternalTable;
import org.apache.doris.datasource.es.source.EsScanNode;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
@@ -624,9 +625,10 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
@Override
public PlanFragment visitPhysicalEsScan(PhysicalEsScan esScan,
PlanTranslatorContext context) {
List<Slot> slots = esScan.getOutput();
- ExternalTable table = esScan.getTable();
+ TableIf table = esScan.getTable();
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table,
context);
- EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(),
tupleDescriptor, true);
+ EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(),
tupleDescriptor,
+ table instanceof EsExternalTable);
esScanNode.setNereidsId(esScan.getId());
esScanNode.addConjuncts(translateToLegacyConjuncts(esScan.getConjuncts()));
Utils.execWithUncheckedException(esScanNode::init);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 67000b3fee9..4e18039c4b1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -33,7 +33,6 @@ import org.apache.doris.common.Config;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.ExternalTable;
-import org.apache.doris.datasource.es.EsExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
import org.apache.doris.nereids.CTEContext;
@@ -416,17 +415,11 @@ public class BindRelation extends OneAnalysisRuleFactory {
case ODBC:
return new
LogicalOdbcScan(unboundRelation.getRelationId(), table,
qualifierWithoutTableName);
case ES_EXTERNAL_TABLE:
- return new LogicalEsScan(unboundRelation.getRelationId(),
(EsExternalTable) table,
- qualifierWithoutTableName);
+ case ELASTICSEARCH:
+ return new LogicalEsScan(unboundRelation.getRelationId(),
table, qualifierWithoutTableName);
case TEST_EXTERNAL_TABLE:
return new
LogicalTestScan(unboundRelation.getRelationId(), table,
qualifierWithoutTableName);
default:
- try {
- // TODO: support other type table, such as
ELASTICSEARCH
-
cascadesContext.getConnectContext().getSessionVariable().enableFallbackToOriginalPlannerOnce();
- } catch (Exception e) {
- // ignore
- }
throw new AnalysisException("Unsupported tableType " +
table.getType());
}
} finally {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java
index ea278aa203c..58954694422 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java
@@ -17,7 +17,7 @@
package org.apache.doris.nereids.trees.plans.logical;
-import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -27,7 +27,6 @@ import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import java.util.List;
@@ -42,22 +41,16 @@ public class LogicalEsScan extends LogicalExternalRelation {
/**
* Constructor for LogicalEsScan.
*/
- public LogicalEsScan(RelationId id, ExternalTable table, List<String>
qualifier,
+ public LogicalEsScan(RelationId id, TableIf table, List<String> qualifier,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties,
Set<Expression> conjuncts) {
super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, conjuncts,
groupExpression, logicalProperties);
}
- public LogicalEsScan(RelationId id, ExternalTable table, List<String>
qualifier) {
+ public LogicalEsScan(RelationId id, TableIf table, List<String> qualifier)
{
this(id, table, qualifier, Optional.empty(), Optional.empty(),
ImmutableSet.of());
}
- @Override
- public ExternalTable getTable() {
- Preconditions.checkArgument(table instanceof ExternalTable);
- return (ExternalTable) table;
- }
-
@Override
public String toString() {
return Utils.toSqlString("LogicalEsScan",
@@ -68,26 +61,26 @@ public class LogicalEsScan extends LogicalExternalRelation {
@Override
public LogicalEsScan withGroupExpression(Optional<GroupExpression>
groupExpression) {
- return new LogicalEsScan(relationId, (ExternalTable) table, qualifier,
groupExpression,
+ return new LogicalEsScan(relationId, table, qualifier, groupExpression,
Optional.of(getLogicalProperties()), conjuncts);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression>
groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan>
children) {
- return new LogicalEsScan(relationId, (ExternalTable) table, qualifier,
groupExpression, logicalProperties,
+ return new LogicalEsScan(relationId, table, qualifier,
groupExpression, logicalProperties,
conjuncts);
}
@Override
public LogicalEsScan withConjuncts(Set<Expression> conjuncts) {
- return new LogicalEsScan(relationId, (ExternalTable) table, qualifier,
Optional.empty(),
+ return new LogicalEsScan(relationId, table, qualifier,
Optional.empty(),
Optional.of(getLogicalProperties()), conjuncts);
}
@Override
public LogicalEsScan withRelationId(RelationId relationId) {
- return new LogicalEsScan(relationId, (ExternalTable) table, qualifier,
Optional.empty(),
+ return new LogicalEsScan(relationId, table, qualifier,
Optional.empty(),
Optional.empty(), conjuncts);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java
index 2f3a399ee23..474436f1dbb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java
@@ -17,7 +17,7 @@
package org.apache.doris.nereids.trees.plans.physical;
-import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DistributionSpec;
import org.apache.doris.nereids.properties.LogicalProperties;
@@ -48,7 +48,7 @@ public class PhysicalEsScan extends PhysicalCatalogRelation {
/**
* Constructor for PhysicalEsScan.
*/
- public PhysicalEsScan(RelationId id, ExternalTable table, List<String>
qualifier,
+ public PhysicalEsScan(RelationId id, TableIf table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression>
groupExpression,
LogicalProperties logicalProperties, Set<Expression> conjuncts) {
super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier,
groupExpression, logicalProperties);
@@ -59,7 +59,7 @@ public class PhysicalEsScan extends PhysicalCatalogRelation {
/**
* Constructor for PhysicalEsScan.
*/
- public PhysicalEsScan(RelationId id, ExternalTable table, List<String>
qualifier,
+ public PhysicalEsScan(RelationId id, TableIf table, List<String> qualifier,
DistributionSpec distributionSpec, Optional<GroupExpression>
groupExpression,
LogicalProperties logicalProperties, PhysicalProperties
physicalProperties, Statistics statistics,
Set<Expression> conjuncts) {
@@ -96,11 +96,6 @@ public class PhysicalEsScan extends PhysicalCatalogRelation {
groupExpression, logicalProperties.get(), conjuncts);
}
- @Override
- public ExternalTable getTable() {
- return (ExternalTable) table;
- }
-
@Override
public PhysicalEsScan withPhysicalPropertiesAndStats(PhysicalProperties
physicalProperties,
Statistics
statsDeriveResult) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/planner/StatisticDeriveTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/StatisticDeriveTest.java
index 0771508d49a..f66f9cb0777 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/StatisticDeriveTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/StatisticDeriveTest.java
@@ -61,18 +61,6 @@ public class StatisticDeriveTest extends TestWithFeService {
+ " \"replication_num\" = \"1\"\n"
+ ");");
- createTable("create external table test.mysql_table\n"
- + "(k1 int, k2 int)\n"
- + "ENGINE=MYSQL\n"
- + "PROPERTIES (\n"
- + "\"host\" = \"127.0.0.1\",\n"
- + "\"port\" = \"3306\",\n"
- + "\"user\" = \"root\",\n"
- + "\"password\" = \"123\",\n"
- + "\"database\" = \"db1\",\n"
- + "\"table\" = \"tbl1\"\n"
- + ");");
-
createTable("create external table test.odbc_oracle\n"
+ "(k1 float, k2 int)\n"
+ "ENGINE=ODBC\n"
@@ -216,21 +204,6 @@ public class StatisticDeriveTest extends TestWithFeService
{
assertSQLPlanOrErrorMsgContains(sql, "HASH JOIN");
}
- @Test
- public void testMysqlScanStatsDerive() throws Exception {
- String sql = "select * from test.mysql_table";
- SessionVariable sessionVariable = connectContext.getSessionVariable();
- sessionVariable.setEnableJoinReorderBasedCost(true);
- sessionVariable.setDisableJoinReorder(false);
- StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql);
- stmtExecutor.execute();
- Assert.assertNotNull(stmtExecutor.planner());
- Assert.assertNotNull(stmtExecutor.planner().getFragments());
- Assert.assertNotEquals(0,
stmtExecutor.planner().getFragments().size());
- System.out.println(getSQLPlanOrErrorMsg("explain " + sql));
- assertSQLPlanOrErrorMsgContains(sql, "SCAN MYSQL");
- }
-
@Test
public void testOdbcScanStatsDerive() throws Exception {
String sql = "select * from test.odbc_mysql";
diff --git
a/regression-test/suites/correctness_p0/test_push_conjunct_external_table.groovy
b/regression-test/suites/correctness_p0/test_push_conjunct_external_table.groovy
deleted file mode 100644
index 008be4f11b9..00000000000
---
a/regression-test/suites/correctness_p0/test_push_conjunct_external_table.groovy
+++ /dev/null
@@ -1,67 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
- // or more contributor license agreements. See the NOTICE file
- // distributed with this work for additional information
- // regarding copyright ownership. The ASF licenses this file
- // to you under the Apache License, Version 2.0 (the
- // "License"); you may not use this file except in compliance
- // with the License. You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing,
- // software distributed under the License is distributed on an
- // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- // KIND, either express or implied. See the License for the
- // specific language governing permissions and limitations
- // under the License.
-
-suite("test_push_conjunct_external_table") {
- sql """ DROP TABLE IF EXISTS dim_server; """
- sql """ admin set frontend config("enable_odbc_mysql_broker_table" =
"true")"""
-
- sql """
- CREATE EXTERNAL TABLE `dim_server` (
- `col1` varchar(50) NOT NULL,
- `col2` varchar(50) NOT NULL
- ) ENGINE=mysql
- PROPERTIES
- (
- "host" = "127.0.0.1",
- "port" = "8239",
- "user" = "mysql_user",
- "password" = "mysql_passwd",
- "database" = "mysql_db_test",
- "table" = "mysql_table_test",
- "charset" = "utf8mb4"
- );
- """
-
- sql """
- DROP view if exists ads_oreo_sid_report;
- """
-
- sql """
- create view ads_oreo_sid_report
- (
- `col1` ,
- `col2`
- )
- AS
- select
- tmp.col1,tmp.col2
- from (
- select 'abc' as col1,'def' as col2
- ) tmp
- inner join dim_server ds on tmp.col1 = ds.col1 and tmp.col2 = ds.col2;
- """
-
- explain {
- sql """select * from ads_oreo_sid_report where col1='abc' and
col2='def';"""
- contains "`ds`.`col1` = 'abc'"
- contains "`ds`.`col2` = 'def'"
- }
-
- sql """ DROP TABLE IF EXISTS dim_server; """
- sql """ DROP view if exists ads_oreo_sid_report; """
-}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
