morrySnow commented on code in PR #66184:
URL: https://github.com/apache/doris/pull/66184#discussion_r3673922788
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java:
##########
@@ -3131,12 +3146,100 @@ private void addConjunctsToPlanNode(PhysicalFilter<?
extends Plan> filter,
PlanTranslatorContext context) {
for (Expression conjunct : filter.getConjuncts()) {
for (Expression singleConjunct :
ExpressionUtils.extractConjunctionToSet(conjunct)) {
-
planNode.addConjunct(ExpressionTranslator.translate(singleConjunct, context));
+ Expr translated =
ExpressionTranslator.translate(singleConjunct, context);
+ if (canEvaluateOnDictionary(singleConjunct)) {
+ translated.setCanDictFilterFromNereids(true);
+ }
+ planNode.addConjunct(translated);
}
}
updateLegacyPlanIdToPhysicalPlan(planNode, filter);
}
+ /**
+ * Whether a filter conjunct is safe to evaluate on a column's dictionary
(distinct
+ * values) rather than per row, letting ORC/Parquet scans run a heavy
string predicate
+ * once per distinct value. This is the authoritative check (BE only has a
conservative
+ * fallback): the predicate must be an equality/IN whose value side is
derived from a
+ * single slot, is deterministic, and preserves NULL semantics.
+ *
+ * <p>NULL preservation is the subtle part. The dictionary carries no NULL
entry, so a
+ * NULL source value keeps its NULL dict-code and the rewritten {@code
dict_code IN
+ * (surviving)} predicate evaluates to NULL (i.e. "not matched") for it.
That only matches
+ * per-row semantics when the value side is itself NULL whenever the
source column is NULL
+ * -- i.e. null-in implies null-out. So {@code substr(col, 1, 3) = 'abc'}
or
+ * {@code split_by_string(col, ',')[1] = 'x'} are safe, but {@code
coalesce(col, 'N') = 'N'},
+ * {@code ifnull}, {@code if}, {@code nullif} and {@code concat_ws} are
not: they can turn a
+ * NULL input into a non-NULL output, so their NULL rows would be wrongly
dropped.
+ */
+ private static boolean canEvaluateOnDictionary(Expression conjunct) {
+ Expression valueSide;
+ if (conjunct instanceof EqualTo) {
+ EqualTo eq = (EqualTo) conjunct;
+ if (eq.right() instanceof Literal) {
+ valueSide = eq.left();
+ } else if (eq.left() instanceof Literal) {
+ valueSide = eq.right();
+ } else {
+ return false;
+ }
+ } else if (conjunct instanceof InPredicate) {
+ InPredicate in = (InPredicate) conjunct;
+ for (Expression option : in.getOptions()) {
+ if (!(option instanceof Literal)) {
+ return false;
+ }
+ }
+ valueSide = in.getCompareExpr();
+ } else {
+ return false;
+ }
+ // Single-column derivation: dictionaries are per column, so the value
side may
+ // reference exactly one slot (a bare column ref, or a function over
one column).
+ if (valueSide.getInputSlots().size() != 1) {
+ return false;
+ }
+ // Deterministic: BE evaluates each distinct value once and caches it,
so rand/uuid
+ // and other non-deterministic functions must not be dict-filtered.
+ if (valueSide.containsNondeterministic()) {
+ return false;
+ }
+ // NULL preservation: the value side must be NULL whenever the source
column is NULL,
+ // otherwise NULL rows (which have no dictionary entry) would be
wrongly filtered out.
+ return isNullPreserving(valueSide);
+ }
+
+ /**
+ * Whether {@code expr} is guaranteed to be NULL whenever any referenced
slot is NULL
+ * (null-in implies null-out). Slot refs and casts propagate NULL;
literals are constant;
+ * a function propagates NULL only if it is {@link PropagateNullable} or
explicitly known to
+ * be null-propagating. Everything else (coalesce/if/nvl/nullif/concat_ws,
unknown builtins,
+ * UDFs, case-when) is rejected so a NULL input can never become a
non-NULL output.
+ */
+ private static boolean isNullPreserving(Expression expr) {
Review Comment:
this should impl on Expression or ExpressionUtils. maybe already have a impl
for same purpose.
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java:
##########
@@ -3131,12 +3146,100 @@ private void addConjunctsToPlanNode(PhysicalFilter<?
extends Plan> filter,
PlanTranslatorContext context) {
for (Expression conjunct : filter.getConjuncts()) {
for (Expression singleConjunct :
ExpressionUtils.extractConjunctionToSet(conjunct)) {
-
planNode.addConjunct(ExpressionTranslator.translate(singleConjunct, context));
+ Expr translated =
ExpressionTranslator.translate(singleConjunct, context);
+ if (canEvaluateOnDictionary(singleConjunct)) {
+ translated.setCanDictFilterFromNereids(true);
+ }
+ planNode.addConjunct(translated);
}
}
updateLegacyPlanIdToPhysicalPlan(planNode, filter);
}
+ /**
+ * Whether a filter conjunct is safe to evaluate on a column's dictionary
(distinct
+ * values) rather than per row, letting ORC/Parquet scans run a heavy
string predicate
+ * once per distinct value. This is the authoritative check (BE only has a
conservative
+ * fallback): the predicate must be an equality/IN whose value side is
derived from a
+ * single slot, is deterministic, and preserves NULL semantics.
+ *
+ * <p>NULL preservation is the subtle part. The dictionary carries no NULL
entry, so a
+ * NULL source value keeps its NULL dict-code and the rewritten {@code
dict_code IN
+ * (surviving)} predicate evaluates to NULL (i.e. "not matched") for it.
That only matches
+ * per-row semantics when the value side is itself NULL whenever the
source column is NULL
+ * -- i.e. null-in implies null-out. So {@code substr(col, 1, 3) = 'abc'}
or
+ * {@code split_by_string(col, ',')[1] = 'x'} are safe, but {@code
coalesce(col, 'N') = 'N'},
+ * {@code ifnull}, {@code if}, {@code nullif} and {@code concat_ws} are
not: they can turn a
+ * NULL input into a non-NULL output, so their NULL rows would be wrongly
dropped.
+ */
+ private static boolean canEvaluateOnDictionary(Expression conjunct) {
Review Comment:
same as isNullPreserving
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
