morrySnow commented on code in PR #67182:
URL: https://github.com/apache/doris/pull/67182#discussion_r3880252846
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MonthCeil.java:
##########
@@ -108,6 +110,15 @@ public RoundingType getRoundingType() {
return RoundingType.CEIL;
}
+ @Override
+ public Optional<Expression> previousBucketBoundary(Literal value) {
+ Optional<Expression> period = regularBucketPeriod();
+ if (!period.isPresent()) {
+ return Optional.empty();
+ }
+ return Optional.of(new MonthsSub(value, period.get()));
Review Comment:
为什么不直接返回literal呢？
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/AddMonotonicFunctionPruningPredicates.java:
##########
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.processor.post;
+
+import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.datasource.plugin.PluginDrivenExternalTable;
+import org.apache.doris.nereids.CascadesContext;
+import
org.apache.doris.nereids.rules.expression.rules.InferPredicateFromMonotonicFunction;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.table.File;
+import org.apache.doris.nereids.trees.expressions.functions.table.Hdfs;
+import org.apache.doris.nereids.trees.expressions.functions.table.Http;
+import org.apache.doris.nereids.trees.expressions.functions.table.Local;
+import org.apache.doris.nereids.trees.expressions.functions.table.S3;
+import
org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
+import
org.apache.doris.nereids.trees.plans.physical.PhysicalStorageLayerAggregate;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * Add bare-column predicates that storage min/max indexes can consume for
OLAP scans, external
+ * catalog file scans, and external-file table-valued functions. This
post-processor runs after CBO
+ * and partition-predicate removal, so the extra predicates improve scan
pruning without changing
+ * cardinality estimation, join planning, or materialized-view matching.
+ */
+public class AddMonotonicFunctionPruningPredicates extends PlanPostProcessor {
+
+ @Override
+ public Plan visitPhysicalFilter(PhysicalFilter<? extends Plan> filter,
CascadesContext context) {
+ filter = (PhysicalFilter<? extends Plan>) super.visit(filter, context);
+ Plan child = filter.child();
+ if (!supportsStoragePruning(child)) {
+ return filter;
+ }
+
+ Set<Expression> rewrittenConjuncts = new LinkedHashSet<>();
+ for (Expression conjunct : filter.getConjuncts()) {
+ Expression rewritten =
InferPredicateFromMonotonicFunction.inferForPruning(conjunct);
+
rewrittenConjuncts.addAll(ExpressionUtils.extractConjunction(rewritten));
+ }
+ if (rewrittenConjuncts.equals(filter.getConjuncts())) {
+ return filter;
+ }
+ return filter.withConjunctsAndChild(rewrittenConjuncts, child)
+ .copyStatsAndGroupIdFrom((AbstractPhysicalPlan) filter);
+ }
+
+ private static boolean supportsStoragePruning(Plan plan) {
+ if (plan instanceof PhysicalStorageLayerAggregate) {
+ plan = ((PhysicalStorageLayerAggregate) plan).getRelation();
+ }
+ if (plan instanceof PhysicalOlapScan) {
+ return true;
+ }
+ if (plan instanceof PhysicalFileScan) {
+ ExternalTable table = ((PhysicalFileScan) plan).getTable();
+ return table instanceof PluginDrivenExternalTable
+ && ((PluginDrivenExternalTable)
table).supportsStoragePredicatePruning();
+ }
+ if (!(plan instanceof PhysicalTVFRelation)) {
+ return false;
+ }
+ // Do not call getCatalogFunction() here: constructing an
external-file TVF may list remote
+ // files. Match the already-bound Nereids function type without adding
post-processing I/O.
+ TableValuedFunction function = ((PhysicalTVFRelation)
plan).getFunction();
+ return function instanceof File || function instanceof Hdfs ||
function instanceof Http
+ || function instanceof Local || function instanceof S3;
Review Comment:
这里建议改成 `if(plan instanceof PhysicalTVFRelation)` 最后用`return false`兜底
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/InferPredicateFromMonotonicFunction.java:
##########
@@ -319,20 +506,40 @@ private static boolean isDateSlot(Expression expression) {
&& !expression.getDataType().isTimeStampTzType();
}
- // Type-coerce the derived predicate and mark it inferred, so downstream
keeps it inside pruning
- // and never writes it back as a user filter.
+ // Type-coerce the derived predicate and mark it inferred, so consumers
can distinguish it from
+ // a predicate written by the user.
private static Expression inferredPredicate(ComparisonPredicate predicate)
{
- return
TypeCoercionUtils.processComparisonPredicate(predicate).withInferred(true);
+ return coerceAndFoldRight(predicate).withInferred(true);
}
// The date floor/ceil binder may promote a DATE slot to DATETIMEV2
because its hidden default
// origin is DATETIMEV2. Derive against that actual argument first, then
reuse the comparison
// simplifier to recover a bare date Slot without losing non-midnight
boundary semantics.
private static Optional<Expression>
inferredDatePredicate(ComparisonPredicate predicate) {
- ComparisonPredicate coerced = (ComparisonPredicate)
TypeCoercionUtils.processComparisonPredicate(predicate);
+ ComparisonPredicate coerced = coerceAndFoldRight(predicate);
Expression simplified = SimplifyComparisonPredicate.simplify(coerced);
return simplified instanceof ComparisonPredicate &&
isDateSlot(simplified.child(0))
+ && simplified.child(1) instanceof Literal
? Optional.of(simplified.withInferred(true))
: Optional.empty();
}
+
+ private static ComparisonPredicate coerceAndFoldRight(ComparisonPredicate
predicate) {
+ ComparisonPredicate coerced = (ComparisonPredicate)
TypeCoercionUtils.processComparisonPredicate(predicate);
+ Optional<Literal> foldedRight = foldLiteral(coerced.right());
+ if (foldedRight.isPresent()) {
+ coerced = (ComparisonPredicate)
coerced.withChildren(coerced.left(), foldedRight.get());
+ }
+ return coerced;
+ }
+
+ private static final class PrefixInfo {
Review Comment:
add comment to explain `length`
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/InferPredicateFromMonotonicFunction.java:
##########
@@ -141,40 +162,54 @@ private static Expression rewriteDisjunction(Or
predicate) {
List<Expression> children = predicate.children();
List<Expression> rewrittenChildren =
Lists.newArrayListWithCapacity(children.size());
for (Expression child : children) {
- Expression rewritten = inferForPartitionPrune(child);
+ Expression rewritten = inferForPruning(child);
rewrittenChildren.add(rewritten);
}
return predicate.withChildren(rewrittenChildren);
}
- // Try the three inference kinds in order. Precondition: the right side
must be a literal.
+ // Normalize the comparison, then try each supported inference family in
order.
private static Optional<Expression> infer(ComparisonPredicate comparison) {
- if (!(comparison.right() instanceof Literal)) {
+ ComparisonPredicate normalized = comparison.left() instanceof Literal
+ && !(comparison.right() instanceof Literal) ?
comparison.commute() : comparison;
+ if (!(normalized.right() instanceof Literal) || normalized.right()
instanceof NullLiteral) {
return Optional.empty();
}
- return inferPrefixPredicate(comparison)
- .or(() -> inferYearPredicate(comparison))
- .or(() -> inferRoundingPredicate(comparison));
+ return inferPrefixPredicate(normalized)
+ .or(() -> inferYearPredicate(normalized))
+ .or(() -> inferDateFormatPredicate(normalized))
+ .or(() -> inferRoundingPredicate(normalized));
}
// Prefix inference: a prefix never sorts after the whole string
(prefix(s) <= s), so
- // prefix(col) >= 'abc' => col >= 'abc'. >/>= keep the operator; = yields
only the lower bound;
- // </<= do not hold (prefix(col) <= 'abc' allows col = 'abd', which is
larger).
+ // prefix(col) >= 'abc' => col >= 'abc'. >/>= keep the operator; equality
also gets an
+ // exclusive successor upper bound when one is safe. </<= do not yield a
source bound.
private static Optional<Expression>
inferPrefixPredicate(ComparisonPredicate comparison) {
Review Comment:
注释里面增加对输入的约束描述
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/InferPredicateFromMonotonicFunction.java:
##########
@@ -141,40 +162,54 @@ private static Expression rewriteDisjunction(Or
predicate) {
List<Expression> children = predicate.children();
List<Expression> rewrittenChildren =
Lists.newArrayListWithCapacity(children.size());
for (Expression child : children) {
- Expression rewritten = inferForPartitionPrune(child);
+ Expression rewritten = inferForPruning(child);
rewrittenChildren.add(rewritten);
}
return predicate.withChildren(rewrittenChildren);
}
- // Try the three inference kinds in order. Precondition: the right side
must be a literal.
+ // Normalize the comparison, then try each supported inference family in
order.
private static Optional<Expression> infer(ComparisonPredicate comparison) {
- if (!(comparison.right() instanceof Literal)) {
+ ComparisonPredicate normalized = comparison.left() instanceof Literal
+ && !(comparison.right() instanceof Literal) ?
comparison.commute() : comparison;
+ if (!(normalized.right() instanceof Literal) || normalized.right()
instanceof NullLiteral) {
return Optional.empty();
}
- return inferPrefixPredicate(comparison)
- .or(() -> inferYearPredicate(comparison))
- .or(() -> inferRoundingPredicate(comparison));
+ return inferPrefixPredicate(normalized)
+ .or(() -> inferYearPredicate(normalized))
+ .or(() -> inferDateFormatPredicate(normalized))
+ .or(() -> inferRoundingPredicate(normalized));
}
// Prefix inference: a prefix never sorts after the whole string
(prefix(s) <= s), so
- // prefix(col) >= 'abc' => col >= 'abc'. >/>= keep the operator; = yields
only the lower bound;
- // </<= do not hold (prefix(col) <= 'abc' allows col = 'abd', which is
larger).
+ // prefix(col) >= 'abc' => col >= 'abc'. >/>= keep the operator; equality
also gets an
+ // exclusive successor upper bound when one is safe. </<= do not yield a
source bound.
private static Optional<Expression>
inferPrefixPredicate(ComparisonPredicate comparison) {
if (!(comparison.right() instanceof StringLikeLiteral)) {
return Optional.empty();
}
- Optional<Expression> source = prefixSource(comparison.left());
- if (!source.isPresent()) {
+ Optional<PrefixInfo> prefix = extractPrefix(comparison.left());
+ if (!prefix.isPresent()) {
return Optional.empty();
}
ComparisonPredicate inferred;
if (comparison instanceof GreaterThan || comparison instanceof
GreaterThanEqual) {
- inferred = (ComparisonPredicate)
comparison.withChildren(source.get(), comparison.right());
+ inferred = (ComparisonPredicate)
comparison.withChildren(prefix.get().source, comparison.right());
} else if (comparison instanceof EqualTo) {
- inferred = new GreaterThanEqual(source.get(), comparison.right());
+ Expression lower = inferredPredicate(new
GreaterThanEqual(prefix.get().source, comparison.right()));
+ String value = ((StringLikeLiteral)
comparison.right()).getStringValue();
+ if (BigDecimal.valueOf(value.codePointCount(0,
value.length())).compareTo(prefix.get().length) > 0) {
Review Comment:
length是算byte数的，所以这里是不是有点儿问题？https://doris.apache.org/docs/dev/sql-manual/sql-functions/scalar-functions/string-functions/length?_highlight=length
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/InferPredicateFromMonotonicFunction.java:
##########
@@ -209,11 +244,34 @@ private static Optional<Expression>
prefixSource(Expression expression) {
|| ((IntegerLikeLiteral) length).getBigDecimalValue().signum()
<= 0) {
return Optional.empty();
}
- return Optional.of(source);
+ return Optional.of(new PrefixInfo(source, ((IntegerLikeLiteral)
length).getBigDecimalValue()));
+ }
+
+ // Build the smallest clean-ASCII string that sorts after every string
with the given prefix.
+ // Non-ASCII bytes are excluded because storage readers do not all use the
same signedness for
+ // byte comparison; retaining only the lower bound is safe in those cases.
+ private static Optional<String> prefixSuccessor(String value) {
+ byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+ for (byte valueByte : bytes) {
+ if ((valueByte & 0xFF) >= 0x80) {
Review Comment:
这个函数里面的16进制数，都需要说明含义
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/InferPredicateFromMonotonicFunction.java:
##########
@@ -141,40 +162,54 @@ private static Expression rewriteDisjunction(Or
predicate) {
List<Expression> children = predicate.children();
List<Expression> rewrittenChildren =
Lists.newArrayListWithCapacity(children.size());
for (Expression child : children) {
- Expression rewritten = inferForPartitionPrune(child);
+ Expression rewritten = inferForPruning(child);
rewrittenChildren.add(rewritten);
}
return predicate.withChildren(rewrittenChildren);
}
- // Try the three inference kinds in order. Precondition: the right side
must be a literal.
+ // Normalize the comparison, then try each supported inference family in
order.
private static Optional<Expression> infer(ComparisonPredicate comparison) {
- if (!(comparison.right() instanceof Literal)) {
+ ComparisonPredicate normalized = comparison.left() instanceof Literal
+ && !(comparison.right() instanceof Literal) ?
comparison.commute() : comparison;
+ if (!(normalized.right() instanceof Literal) || normalized.right()
instanceof NullLiteral) {
return Optional.empty();
}
- return inferPrefixPredicate(comparison)
- .or(() -> inferYearPredicate(comparison))
- .or(() -> inferRoundingPredicate(comparison));
+ return inferPrefixPredicate(normalized)
+ .or(() -> inferYearPredicate(normalized))
+ .or(() -> inferDateFormatPredicate(normalized))
+ .or(() -> inferRoundingPredicate(normalized));
}
// Prefix inference: a prefix never sorts after the whole string
(prefix(s) <= s), so
- // prefix(col) >= 'abc' => col >= 'abc'. >/>= keep the operator; = yields
only the lower bound;
- // </<= do not hold (prefix(col) <= 'abc' allows col = 'abd', which is
larger).
+ // prefix(col) >= 'abc' => col >= 'abc'. >/>= keep the operator; equality
also gets an
+ // exclusive successor upper bound when one is safe. </<= do not yield a
source bound.
private static Optional<Expression>
inferPrefixPredicate(ComparisonPredicate comparison) {
if (!(comparison.right() instanceof StringLikeLiteral)) {
return Optional.empty();
}
- Optional<Expression> source = prefixSource(comparison.left());
- if (!source.isPresent()) {
+ Optional<PrefixInfo> prefix = extractPrefix(comparison.left());
+ if (!prefix.isPresent()) {
return Optional.empty();
}
Review Comment:
这里直接增加两个变量，分别是source和length吧
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
