This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5d9678700c [feature](Nereids) support select tablets with nereids
optimizer (#23164)
5d9678700c is described below
commit 5d9678700cd5a599564f1a51ef3b7b2408dac4f1
Author: Tiewei Fang <[email protected]>
AuthorDate: Tue Aug 22 10:14:27 2023 +0800
[feature](Nereids) support select tablets with nereids optimizer (#23164)
---
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 6 +-
.../doris/nereids/analyzer/UnboundRelation.java | 25 ++++--
.../doris/nereids/parser/LogicalPlanBuilder.java | 9 ++-
.../doris/nereids/rules/analysis/BindRelation.java | 6 +-
.../trees/copier/LogicalPlanDeepCopier.java | 6 +-
.../trees/plans/logical/LogicalOlapScan.java | 10 +--
.../select_tablets/select_with_tablets.out | 31 ++++++++
.../select_tablets/select_with_tablets.groovy | 90 ++++++++++++++++++++++
9 files changed, 166 insertions(+), 18 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 9eb05c1230..95c21ff352 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -356,6 +356,7 @@ SYSTEM_TIME: 'SYSTEM_TIME';
SYSTEM_VERSION: 'SYSTEM_VERSION';
TABLE: 'TABLE';
TABLES: 'TABLES';
+TABLET: 'TABLET';
TABLESAMPLE: 'TABLESAMPLE';
TBLPROPERTIES: 'TBLPROPERTIES';
TEMPORARY: 'TEMPORARY' | 'TEMP';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 1348bb80e6..111d30f394 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -262,7 +262,7 @@ identifierSeq
;
relationPrimary
- : multipartIdentifier specifiedPartition? tableAlias relationHint?
lateralView* #tableName
+ : multipartIdentifier specifiedPartition? tabletList? tableAlias
relationHint? lateralView* #tableName
| LEFT_PAREN query RIGHT_PAREN tableAlias lateralView*
#aliasedQuery
| tvfName=identifier LEFT_PAREN
(properties+=tvfProperty (COMMA properties+=tvfProperty)*)?
@@ -283,6 +283,10 @@ multipartIdentifier
: parts+=errorCapturingIdentifier (DOT parts+=errorCapturingIdentifier)*
;
+tabletList
+ : TABLET LEFT_PAREN tabletIdList+=INTEGER_VALUE (COMMA
tabletIdList+=INTEGER_VALUE)* RIGHT_PAREN
+ ;
+
// -----------------Expression-----------------
namedExpression
: expression (AS? (identifierOrText))?
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java
index 6d625f7068..ceb4a8548e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java
@@ -46,28 +46,35 @@ public class UnboundRelation extends LogicalRelation
implements Unbound {
private final List<String> nameParts;
private final List<String> partNames;
+ private final List<Long> tabletIds;
private final boolean isTempPart;
private final List<String> hints;
public UnboundRelation(RelationId id, List<String> nameParts) {
- this(id, nameParts, Optional.empty(), Optional.empty(),
ImmutableList.of(), false, ImmutableList.of());
+ this(id, nameParts, Optional.empty(), Optional.empty(),
ImmutableList.of(), false, ImmutableList.of(),
+ ImmutableList.of());
}
public UnboundRelation(RelationId id, List<String> nameParts, List<String>
partNames, boolean isTempPart) {
- this(id, nameParts, Optional.empty(), Optional.empty(), partNames,
isTempPart, ImmutableList.of());
+ this(id, nameParts, Optional.empty(), Optional.empty(), partNames,
isTempPart, ImmutableList.of(),
+ ImmutableList.of());
}
public UnboundRelation(RelationId id, List<String> nameParts, List<String>
partNames, boolean isTempPart,
- List<String> hints) {
- this(id, nameParts, Optional.empty(), Optional.empty(), partNames,
isTempPart, hints);
+ List<Long> tabletIds, List<String> hints) {
+ this(id, nameParts, Optional.empty(), Optional.empty(), partNames,
isTempPart, tabletIds, hints);
}
+ /**
+ * constructor of UnboundRelation
+ */
public UnboundRelation(RelationId id, List<String> nameParts,
Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<String>
partNames, boolean isTempPart,
- List<String> hints) {
+ List<Long> tabletIds, List<String> hints) {
super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression,
logicalProperties);
this.nameParts =
ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not
null"));
this.partNames =
ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not
null"));
+ this.tabletIds =
ImmutableList.copyOf(Objects.requireNonNull(tabletIds, "tabletIds should not
null"));
this.isTempPart = isTempPart;
this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints
should not be null."));
}
@@ -90,14 +97,14 @@ public class UnboundRelation extends LogicalRelation
implements Unbound {
public Plan withGroupExpression(Optional<GroupExpression> groupExpression)
{
return new UnboundRelation(relationId, nameParts,
groupExpression, Optional.of(getLogicalProperties()),
- partNames, isTempPart, hints);
+ partNames, isTempPart, tabletIds, hints);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression>
groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan>
children) {
return new UnboundRelation(relationId, nameParts, groupExpression,
logicalProperties, partNames,
- isTempPart, hints);
+ isTempPart, tabletIds, hints);
}
@Override
@@ -136,6 +143,10 @@ public class UnboundRelation extends LogicalRelation
implements Unbound {
return isTempPart;
}
+ public List<Long> getTabletIds() {
+ return tabletIds;
+ }
+
public List<String> getHints() {
return hints;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 876ca00a32..6cd63ad434 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -589,6 +589,13 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
}
}
+ List<Long> tabletIdLists = new ArrayList<>();
+ if (ctx.tabletList() != null) {
+ ctx.tabletList().tabletIdList.stream().forEach(tabletToken -> {
+ tabletIdLists.add(Long.parseLong(tabletToken.getText()));
+ });
+ }
+
final List<String> relationHints;
if (ctx.relationHint() != null) {
relationHints = typedVisit(ctx.relationHint());
@@ -598,7 +605,7 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
LogicalPlan checkedRelation = withCheckPolicy(
new UnboundRelation(StatementScopeIdGenerator.newRelationId(),
- tableId, partitionNames, isTempPart, relationHints));
+ tableId, partitionNames, isTempPart, tabletIdLists,
relationHints));
LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias());
for (LateralViewContext lateralViewContext : ctx.lateralView()) {
plan = withGenerate(plan, lateralViewContext);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index a6b0b34e8d..1fb886c28d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -168,12 +168,14 @@ public class BindRelation extends OneAnalysisRuleFactory {
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation
unboundRelation, List<String> tableQualifier) {
LogicalOlapScan scan;
List<Long> partIds = getPartitionIds(table, unboundRelation);
+ List<Long> tabletIds = unboundRelation.getTabletIds();
if (!CollectionUtils.isEmpty(partIds)) {
scan = new LogicalOlapScan(unboundRelation.getRelationId(),
- (OlapTable) table,
ImmutableList.of(tableQualifier.get(1)), partIds, unboundRelation.getHints());
+ (OlapTable) table,
ImmutableList.of(tableQualifier.get(1)), partIds,
+ tabletIds, unboundRelation.getHints());
} else {
scan = new LogicalOlapScan(unboundRelation.getRelationId(),
- (OlapTable) table,
ImmutableList.of(tableQualifier.get(1)), unboundRelation.getHints());
+ (OlapTable) table,
ImmutableList.of(tableQualifier.get(1)), tabletIds, unboundRelation.getHints());
}
if (!Util.showHiddenColumns() && scan.getTable().hasDeleteSign()
&& !ConnectContext.get().getSessionVariable()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
index 24c839ad12..7fed96245f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
@@ -160,11 +160,13 @@ public class LogicalPlanDeepCopier extends
DefaultPlanRewriter<DeepCopierContext
LogicalOlapScan newOlapScan;
if (olapScan.getManuallySpecifiedPartitions().isEmpty()) {
newOlapScan = new
LogicalOlapScan(StatementScopeIdGenerator.newRelationId(),
- olapScan.getTable(), olapScan.getQualifier(),
olapScan.getHints());
+ olapScan.getTable(), olapScan.getQualifier(),
olapScan.getSelectedTabletIds(),
+ olapScan.getHints());
} else {
newOlapScan = new
LogicalOlapScan(StatementScopeIdGenerator.newRelationId(),
olapScan.getTable(), olapScan.getQualifier(),
- olapScan.getManuallySpecifiedPartitions(),
olapScan.getHints());
+ olapScan.getManuallySpecifiedPartitions(),
olapScan.getSelectedTabletIds(),
+ olapScan.getHints());
}
newOlapScan.getOutput();
context.putRelation(olapScan.getRelationId(), newOlapScan);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 4777e2b65b..9945bb5606 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -113,18 +113,18 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan
-1, false, PreAggStatus.on(), ImmutableList.of(),
ImmutableList.of(), Maps.newHashMap());
}
- public LogicalOlapScan(RelationId id, OlapTable table, List<String>
qualifier, List<String> hints) {
+ public LogicalOlapScan(RelationId id, OlapTable table, List<String>
qualifier, List<Long> tabletIds,
+ List<String> hints) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
- table.getPartitionIds(), false,
- ImmutableList.of(),
+ table.getPartitionIds(), false, tabletIds,
-1, false, PreAggStatus.on(), ImmutableList.of(), hints,
Maps.newHashMap());
}
public LogicalOlapScan(RelationId id, OlapTable table, List<String>
qualifier, List<Long> specifiedPartitions,
- List<String> hints) {
+ List<Long> tabletIds, List<String> hints) {
this(id, table, qualifier, Optional.empty(), Optional.empty(),
// must use specifiedPartitions here for prune partition by
sql like 'select * from t partition p1'
- specifiedPartitions, false, ImmutableList.of(),
+ specifiedPartitions, false, tabletIds,
-1, false, PreAggStatus.on(), specifiedPartitions, hints,
Maps.newHashMap());
}
diff --git
a/regression-test/data/nereids_p0/select_tablets/select_with_tablets.out
b/regression-test/data/nereids_p0/select_tablets/select_with_tablets.out
new file mode 100644
index 0000000000..32863d03d7
--- /dev/null
+++ b/regression-test/data/nereids_p0/select_tablets/select_with_tablets.out
@@ -0,0 +1,31 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select1 --
+1 doris 19
+2 nereids 18
+
+-- !select2 --
+1 doris 19
+2 nereids 18
+
+-- !select3 --
+1 doris 19
+2 nereids 18
+
+-- !select4 --
+
+-- !select5 --
+
+-- !select6 --
+
+-- !no_partition_1 --
+1 doris 19
+2 nereids 18
+
+-- !no_partition_2 --
+2 nereids 18
+
+-- !no_partition_3 --
+1 doris 19
+
+-- !no_partition_4 --
+
diff --git
a/regression-test/suites/nereids_p0/select_tablets/select_with_tablets.groovy
b/regression-test/suites/nereids_p0/select_tablets/select_with_tablets.groovy
new file mode 100644
index 0000000000..2a16511001
--- /dev/null
+++
b/regression-test/suites/nereids_p0/select_tablets/select_with_tablets.groovy
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("select_with_tablets") {
+ sql 'set enable_nereids_planner=true;'
+ sql 'set enable_fallback_to_original_planner=false;'
+
+
+ def table_name1 = "test_table"
+ sql """ DROP TABLE IF EXISTS ${table_name1} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_name1} (
+ `id` int(11) NULL,
+ `name` string NULL,
+ `age` int(11) NULL
+ )
+ PARTITION BY RANGE(id)
+ (
+ PARTITION less_than_20 VALUES LESS THAN ("20"),
+ PARTITION between_20_70 VALUES [("20"),("70")),
+ PARTITION more_than_70 VALUES LESS THAN ("151")
+ )
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${table_name1} VALUES (1, 'doris', 19), (2, 'nereids',
18) """
+ def insert_res = sql "show last insert;"
+
+ logger.info("insert result: " + insert_res.toString())
+ order_qt_select1 """ SELECT * FROM ${table_name1} """
+
+ def res = sql """ show tablets from ${table_name1} where version = 2 """
+ assertTrue(res.size() == 1)
+ assertTrue(res[0].size() == 21)
+ assertEquals("2", res[0][4])
+
+ order_qt_select2 """ SELECT * FROM ${table_name1} TABLET(${res[0][0]}) """
+ order_qt_select3 """ SELECT * FROM ${table_name1} PARTITION less_than_20
TABLET(${res[0][0]}) """
+ // result should be empty because TABLET(${res[0][0]}) is not belonged to
partition between_20_70.
+ order_qt_select4 """ SELECT * FROM ${table_name1} PARTITION between_20_70
TABLET(${res[0][0]}) """
+
+ res = sql """ show tablets from ${table_name1} where version = 1 """
+ assertTrue(res.size() == 2)
+ assertEquals("1", res[0][4])
+ assertEquals("1", res[1][4])
+ // result should be empty because TABLET(${res[0][0]}) does not have data.
+ order_qt_select5 """ SELECT * FROM ${table_name1} TABLET(${res[0][0]}) """
+ // result should be empty because TABLET(${res[1][0]}) does not have data.
+ order_qt_select6 """ SELECT * FROM ${table_name1} TABLET(${res[1][0]}) """
+
+ // Test non-partitioned table
+ def table_no_partition = "table_no_partition"
+ sql """ DROP TABLE IF EXISTS ${table_no_partition} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_no_partition} (
+ `id` int(11) NULL,
+ `name` string NULL,
+ `age` int(11) NULL
+ )
+ DISTRIBUTED BY HASH(id) BUCKETS 3
+ PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ INSERT INTO ${table_no_partition} VALUES (1, 'doris', 19), (2,
'nereids', 18) """
+ insert_res = sql "show last insert;"
+
+ logger.info("insert result: " + insert_res.toString())
+ order_qt_no_partition_1 """ SELECT * FROM ${table_no_partition} """
+
+ res = sql """ show tablets from ${table_no_partition} where version = 2 """
+
+ order_qt_no_partition_2 """ SELECT * FROM ${table_no_partition}
TABLET(${res[0][0]}) """
+ order_qt_no_partition_3 """ SELECT * FROM ${table_no_partition}
TABLET(${res[1][0]}) """
+ order_qt_no_partition_4 """ SELECT * FROM ${table_no_partition}
TABLET(${res[2][0]}) """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
