This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 281a84587b [fix](planner) conjuncts of the outer query block didn't
work when it's on the results expr of inline view (#17203)
281a84587b is described below
commit 281a84587beea4423e7bc0f73cdc0b62dbd5ecb7
Author: AKIRA <[email protected]>
AuthorDate: Mon Feb 27 22:51:43 2023 +0900
[fix](planner) conjuncts of the outer query block didn't work when it's on
the results expr of inline view (#17203)
cherry-pick #17036
---
.../java/org/apache/doris/analysis/Analyzer.java | 20 +++++++++++---
.../org/apache/doris/analysis/BinaryPredicate.java | 4 +--
.../java/org/apache/doris/analysis/CastExpr.java | 4 +--
.../apache/doris/analysis/CompoundPredicate.java | 4 +--
.../main/java/org/apache/doris/analysis/Expr.java | 8 +++---
.../apache/doris/analysis/FunctionCallExpr.java | 2 +-
.../org/apache/doris/analysis/InPredicate.java | 4 +--
.../org/apache/doris/analysis/IsNullPredicate.java | 4 +--
.../java/org/apache/doris/analysis/SetVar.java | 2 +-
.../java/org/apache/doris/analysis/SlotRef.java | 22 +++++++++++++++
.../org/apache/doris/analysis/SysVariableDesc.java | 4 +--
.../java/org/apache/doris/planner/PlanNode.java | 8 ++++++
.../java/org/apache/doris/planner/SelectNode.java | 6 -----
.../apache/doris/planner/SingleNodePlanner.java | 13 ++++++---
.../apache/doris/rewrite/FoldConstantsRule.java | 2 +-
.../data/query_p0/literal_view/lietral_test.out | 2 ++
regression-test/data/query_p0/union/test_union.out | 3 +++
.../query_p0/literal_view/lietral_test.groovy | 31 ++++++++++++++++++++++
.../suites/query_p0/union/test_union.groovy | 7 +++++
19 files changed, 119 insertions(+), 31 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
index 50a9fd7ce0..3165f5ba12 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -396,6 +396,8 @@ public class Analyzer {
private final Set<TupleId> markTupleIdsNotProcessed =
Sets.newHashSet();
+ private final Map<InlineViewRef, Set<Expr>> migrateFailedConjuncts =
Maps.newHashMap();
+
public GlobalState(Env env, ConnectContext context) {
this.env = env;
this.context = context;
@@ -1184,6 +1186,16 @@ public class Analyzer {
}
}
+ public void registerMigrateFailedConjuncts(InlineViewRef ref, Expr e)
throws AnalysisException {
+ markConstantConjunct(e, false);
+ Set<Expr> exprSet =
globalState.migrateFailedConjuncts.computeIfAbsent(ref, (k) -> new HashSet<>());
+ exprSet.add(e);
+ }
+
+ public Set<Expr> findMigrateFailedConjuncts(InlineViewRef inlineViewRef) {
+ return globalState.migrateFailedConjuncts.get(inlineViewRef);
+ }
+
/**
* register expr id
* @param expr
@@ -1340,7 +1352,8 @@ public class Analyzer {
&& !e.isAuxExpr()
&& !globalState.assignedConjuncts.contains(e.getId())
&& ((inclOjConjuncts && !e.isConstant())
- ||
!globalState.ojClauseByConjunct.containsKey(e.getId()))) {
+ || (!globalState.ojClauseByConjunct.containsKey(e.getId())
+ &&
!globalState.sjClauseByConjunct.containsKey(e.getId())))) {
result.add(e);
}
}
@@ -1817,7 +1830,8 @@ public class Analyzer {
// aliases and having it analyzed is needed for the
following EvalPredicate() call
conjunct.analyze(this);
}
- final Expr newConjunct = conjunct.getResultValue();
+ Expr newConjunct = conjunct.getResultValue(true);
+ newConjunct = FoldConstantsRule.INSTANCE.apply(newConjunct,
this, null);
if (newConjunct instanceof BoolLiteral || newConjunct
instanceof NullLiteral) {
boolean evalResult = true;
if (newConjunct instanceof BoolLiteral) {
@@ -2390,7 +2404,7 @@ public class Analyzer {
if (tids.size() > 1 ||
globalState.ojClauseByConjunct.containsKey(e.getId())
|| globalState.outerJoinedTupleIds.containsKey(e.getId()) &&
whereClauseConjuncts.contains(e.getId())
- || globalState.conjunctsByOjClause.containsKey(e.getId())) {
+ || globalState.conjunctsByOjClause.containsKey(e.getId())) {
return true;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
index fed13dad19..2b01f17319 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
@@ -647,8 +647,8 @@ public class BinaryPredicate extends Predicate implements
Writable {
}
@Override
- public Expr getResultValue() throws AnalysisException {
- recursiveResetChildrenResult();
+ public Expr getResultValue(boolean inView) throws AnalysisException {
+ recursiveResetChildrenResult(inView);
final Expr leftChildValue = getChild(0);
final Expr rightChildValue = getChild(1);
if (!(leftChildValue instanceof LiteralExpr)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
index c20cc74908..486e14ac72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -398,8 +398,8 @@ public class CastExpr extends Expr {
}
@Override
- public Expr getResultValue() throws AnalysisException {
- recursiveResetChildrenResult();
+ public Expr getResultValue(boolean inView) throws AnalysisException {
+ recursiveResetChildrenResult(inView);
final Expr value = children.get(0);
if (!(value instanceof LiteralExpr)) {
return this;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
index f1e41ad0b4..8c3247479d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
@@ -235,8 +235,8 @@ public class CompoundPredicate extends Predicate {
}
@Override
- public Expr getResultValue() throws AnalysisException {
- recursiveResetChildrenResult();
+ public Expr getResultValue(boolean inView) throws AnalysisException {
+ recursiveResetChildrenResult(inView);
boolean compoundResult = false;
if (op == Operator.NOT) {
final Expr childValue = getChild(0);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index b7915d207e..37759a5b49 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -1906,10 +1906,10 @@ public abstract class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
}
- protected void recursiveResetChildrenResult() throws AnalysisException {
+ protected void recursiveResetChildrenResult(boolean inView) throws
AnalysisException {
for (int i = 0; i < children.size(); i++) {
final Expr child = children.get(i);
- final Expr newChild = child.getResultValue();
+ final Expr newChild = child.getResultValue(inView);
if (newChild != child) {
setChild(i, newChild);
}
@@ -1921,8 +1921,8 @@ public abstract class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
* @return value returned can't be null, if this and it's children are't
constant expr, return this.
* @throws AnalysisException
*/
- public Expr getResultValue() throws AnalysisException {
- recursiveResetChildrenResult();
+ public Expr getResultValue(boolean forPushDownPredicatesToView) throws
AnalysisException {
+ recursiveResetChildrenResult(forPushDownPredicatesToView);
final Expr newExpr = ExpressionFunctions.INSTANCE.evalExpr(this);
return newExpr != null ? newExpr : this;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index 62373e619a..ee1fd3d869 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -1401,7 +1401,7 @@ public class FunctionCallExpr extends Expr {
* Return type is DATETIME
*/
if (fn.getFunctionName().getFunction().equals("str_to_date")) {
- Expr child1Result = getChild(1).getResultValue();
+ Expr child1Result = getChild(1).getResultValue(false);
if (child1Result instanceof StringLiteral) {
if (DateLiteral.hasTimePart(((StringLiteral)
child1Result).getStringValue())) {
this.type = ScalarType.getDefaultDateType(Type.DATETIME);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java
index 34b8945a4c..e041549ab7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java
@@ -291,8 +291,8 @@ public class InPredicate extends Predicate {
}
@Override
- public Expr getResultValue() throws AnalysisException {
- recursiveResetChildrenResult();
+ public Expr getResultValue(boolean inView) throws AnalysisException {
+ recursiveResetChildrenResult(inView);
final Expr leftChildValue = getChild(0);
if (!(leftChildValue instanceof LiteralExpr) || !isLiteralChildren()) {
return this;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java
index 00f01cc299..572c778262 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java
@@ -154,8 +154,8 @@ public class IsNullPredicate extends Predicate {
* fix issue 6390
*/
@Override
- public Expr getResultValue() throws AnalysisException {
- recursiveResetChildrenResult();
+ public Expr getResultValue(boolean inView) throws AnalysisException {
+ recursiveResetChildrenResult(inView);
final Expr childValue = getChild(0);
if (!(childValue instanceof LiteralExpr)) {
return this;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
index b49423b002..ebe38983f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
@@ -109,7 +109,7 @@ public class SetVar {
throw new AnalysisException("Set statement does't support
non-constant expr.");
}
- final Expr literalExpr = value.getResultValue();
+ final Expr literalExpr = value.getResultValue(false);
if (!(literalExpr instanceof LiteralExpr)) {
throw new AnalysisException("Set statement does't support
computing expr:" + literalExpr.toSql());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
index bfd9ab9ccb..c2ca4a6b6a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
@@ -501,4 +501,26 @@ public class SlotRef extends Expr {
}
return builder.toString();
}
+
+ @Override
+ public Expr getResultValue(boolean foldSlot) throws AnalysisException {
+ if (!foldSlot) {
+ return this;
+ }
+ if (!isConstant() || desc == null) {
+ return this;
+ }
+ List<Expr> exprs = desc.getSourceExprs();
+ if (CollectionUtils.isEmpty(exprs)) {
+ return this;
+ }
+ Expr expr = exprs.get(0);
+ if (expr instanceof SlotRef) {
+ return expr.getResultValue(foldSlot);
+ }
+ if (expr.isConstant()) {
+ return expr;
+ }
+ return this;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/SysVariableDesc.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SysVariableDesc.java
index 29e6a3c18d..023be528e6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SysVariableDesc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SysVariableDesc.java
@@ -116,7 +116,7 @@ public class SysVariableDesc extends Expr {
}
@Override
- public Expr getResultValue() throws AnalysisException {
+ public Expr getResultValue(boolean inView) throws AnalysisException {
if (!Strings.isNullOrEmpty(name) &&
VariableVarConverters.hasConverter(name)) {
// Return the string type here so that it can correctly match the
subsequent function signature.
// And we also set `beConverted` to session variable name in
StringLiteral, so that it can be cast back
@@ -129,7 +129,7 @@ public class SysVariableDesc extends Expr {
throw new AnalysisException(e.getMessage());
}
}
- return super.getResultValue();
+ return super.getResultValue(false);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index fd861842ce..2e58a19ed3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -1120,4 +1120,12 @@ public abstract class PlanNode extends
TreeNode<PlanNode> implements PlanStats {
public List<Expr> getProjectList() {
return projectList;
}
+
+ public List<SlotId> getOutputSlotIds() {
+ return outputSlotIds;
+ }
+
+ public void setVConjunct(Set<Expr> exprs) {
+ vconjunct = convertConjunctsToAndCompoundPredicate(new
ArrayList<>(exprs));
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java
index d46b0310d5..2baa6e6f13 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java
@@ -40,12 +40,6 @@ import java.util.List;
public class SelectNode extends PlanNode {
private static final Logger LOG = LogManager.getLogger(SelectNode.class);
- protected SelectNode(PlanNodeId id, PlanNode child) {
- super(id, child.getTupleIds(), "SELECT", StatisticalType.SELECT_NODE);
- addChild(child);
- this.nullableTupleIds = child.nullableTupleIds;
- }
-
protected SelectNode(PlanNodeId id, PlanNode child, List<Expr> conjuncts) {
super(id, child.getTupleIds(), "SELECT", StatisticalType.SELECT_NODE);
addChild(child);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index d21b44c7e5..95bf5ae804 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -1802,8 +1802,9 @@ public class SingleNodePlanner {
newConjuncts = cloneExprs(newConjuncts);
}
} else {
- Preconditions.checkArgument(select.getTableRefs().size() == 1);
- viewAnalyzer.registerConjuncts(newConjuncts,
select.getTableRefs().get(0).getId());
+ for (Expr e : conjuncts) {
+ viewAnalyzer.registerMigrateFailedConjuncts(inlineViewRef,
e);
+ }
}
} else {
Preconditions.checkArgument(stmt instanceof SetOperationStmt);
@@ -2139,7 +2140,13 @@ public class SingleNodePlanner {
scanNode = createScanNode(analyzer, tblRef, selectStmt);
}
if (tblRef instanceof InlineViewRef) {
- scanNode = createInlineViewPlan(analyzer, (InlineViewRef) tblRef);
+ InlineViewRef inlineViewRef = (InlineViewRef) tblRef;
+ scanNode = createInlineViewPlan(analyzer, inlineViewRef);
+ Analyzer viewAnalyzer = inlineViewRef.getAnalyzer();
+ Set<Expr> exprs =
viewAnalyzer.findMigrateFailedConjuncts(inlineViewRef);
+ if (CollectionUtils.isNotEmpty(exprs)) {
+ scanNode.setVConjunct(exprs);
+ }
}
if (scanNode == null) {
throw new UserException("unknown TableRef node");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
index d812146a35..cff0754b4e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
@@ -122,7 +122,7 @@ public class FoldConstantsRule implements ExprRewriteRule {
return expr;
}
}
- return expr.getResultValue();
+ return expr.getResultValue(false);
}
/**
diff --git a/regression-test/data/query_p0/literal_view/lietral_test.out
b/regression-test/data/query_p0/literal_view/lietral_test.out
index 9c9c4c6c8a..daa91d59cd 100644
--- a/regression-test/data/query_p0/literal_view/lietral_test.out
+++ b/regression-test/data/query_p0/literal_view/lietral_test.out
@@ -1,3 +1,5 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !sql --
+-- !sql1 --
+
diff --git a/regression-test/data/query_p0/union/test_union.out
b/regression-test/data/query_p0/union/test_union.out
index fe0f9c1810..4d398f2f2a 100644
--- a/regression-test/data/query_p0/union/test_union.out
+++ b/regression-test/data/query_p0/union/test_union.out
@@ -364,3 +364,6 @@ hell0
2016-07-01
2016-07-02
+-- !union36 --
+1 2
+
diff --git a/regression-test/suites/query_p0/literal_view/lietral_test.groovy
b/regression-test/suites/query_p0/literal_view/lietral_test.groovy
index 1db3f6d720..0307b0fce8 100644
--- a/regression-test/suites/query_p0/literal_view/lietral_test.groovy
+++ b/regression-test/suites/query_p0/literal_view/lietral_test.groovy
@@ -85,4 +85,35 @@ suite("literal_view_test") {
FROM test_v
WHERE substring('2022-12',6,2)='01';
"""
+
+ sql """DROP TABLE IF EXISTS `test_insert`"""
+
+ sql """
+ CREATE TABLE `test_insert` (
+ `id` varchar(11) NULL COMMENT '唯一标识',
+ `name` varchar(10) NULL COMMENT '采集时间',
+ `age` int(11) NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(`id`)
+ COMMENT 'test'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """
+ insert into test_insert values
(1,'doris',10),(2,'spark',2),(3,'flink',20);
+ """
+
+ qt_sql1 """
+ select id, name
+ from (
+ select '123' as id,
+ '1234' as name,
+ age
+ from test_insert
+ ) a
+ where name != '1234';
+ """
}
diff --git a/regression-test/suites/query_p0/union/test_union.groovy
b/regression-test/suites/query_p0/union/test_union.groovy
index ff856245f0..676d18653a 100644
--- a/regression-test/suites/query_p0/union/test_union.groovy
+++ b/regression-test/suites/query_p0/union/test_union.groovy
@@ -273,4 +273,11 @@ suite("test_union") {
qt_union40 """(select k1 from ${new_union_table}) union (select
k${idx} from ${tbName1}) order by k1"""
}
sql"""drop table ${new_union_table}"""
+
+
+ sql 'set enable_fallback_to_original_planner=false'
+ sql 'set enable_nereids_planner=true'
+ qt_union35 """select cast("2016-07-01" as date) union (select
cast("2016-07-02 1:10:0" as date)) order by 1"""
+
+ qt_union36 """SELECT a,2 as a FROM (SELECT '1' as a) b where a=1;"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
