uros-b commented on code in PR #58484:
URL: https://github.com/apache/spark/pull/58484#discussion_r3966523535
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala:
##########
@@ -881,7 +923,10 @@ object LikeSimplification extends Rule[LogicalPlan] with
PredicateHelper {
// If pattern is null, return null value directly, since "col like
null" == null.
Literal(null, BooleanType)
} else {
- simplifyLike(input, pattern.toString, escapeChar).getOrElse(l)
+ val patternStr = pattern.toString
+ simplifyLike(input, patternStr, escapeChar)
+ .orElse(derivePrefixStartsWith(input, patternStr, escapeChar, l))
Review Comment:
This duplicates `input` (`StartsWith(input, prefix) && Like(input, ...)`).
`LikeAll` already refuses that unless `CollapseProject.isCheap(child)`
(SPARK-40228). Please gate this derive path the same way — otherwise `rand()
LIKE 'a%b%'` evaluates two different `Rand` values, and expensive kids (e.g.
`sha2`) run twice. `'a%'` stays single-eval today; this extends duplication to
`'a%b%'`, `'a_b%'`, etc. A sibling of the existing SPARK-40228 cheap-child test
would lock it in.
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala:
##########
@@ -1751,6 +1751,26 @@ abstract class ParquetFilterSuite extends ParquetTest
with SharedSparkSession {
}
}
+ test("filter pushdown - leading-literal LIKE derives a StartsWith prefix
filter") {
Review Comment:
This only checks that an alphabetic prefix can drop every row group of digit
strings. A bug that dropped the residual `LIKE` and kept only `StartsWith`
would still pass. Can we also assert the pushed filter is `StringStartsWith`,
and that a prefix hit which fails the rest of the pattern (`'abXYZ' LIKE
'ab%cd%'`) is rejected?
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala:
##########
@@ -310,6 +310,57 @@ class LikeSimplificationSuite extends PlanTest {
comparePlans(Optimize.execute(originalQuery), originalQuery)
}
+ test("derive StartsWith prefix guard for leading-literal LIKE 'a%b%'") {
+ val originalQuery = testRelation.where($"a" like "a%b%")
+ val optimized = Optimize.execute(originalQuery.analyze)
+ val correctAnswer = testRelation
+ .where(StartsWith($"a", "a") && ($"a" like "a%b%"))
+ .analyze
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("derive StartsWith prefix guard with a multi-char prefix and multiple
wildcards") {
+ val originalQuery = testRelation.where($"a" like "ab%cd%ef")
+ val optimized = Optimize.execute(originalQuery.analyze)
+ val correctAnswer = testRelation
+ .where(StartsWith($"a", "ab") && ($"a" like "ab%cd%ef"))
+ .analyze
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("derive StartsWith prefix guard when the pattern uses '_' wildcards") {
+ val originalQuery = testRelation.where($"a" like "a_b%")
+ val optimized = Optimize.execute(originalQuery.analyze)
+ val correctAnswer = testRelation
+ .where(StartsWith($"a", "a") && ($"a" like "a_b%"))
+ .analyze
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("no StartsWith prefix guard when the pattern has no leading literal") {
+ val originalQuery = testRelation.where($"a" like "%b%c%").analyze
+ comparePlans(Optimize.execute(originalQuery), originalQuery)
+ }
+
+ test("no StartsWith prefix guard when the pattern contains the escape char")
{
+ val originalQuery = testRelation.where($"a" like "a\\%b%").analyze
+ comparePlans(Optimize.execute(originalQuery), originalQuery)
+ }
+
+ test("no StartsWith prefix guard for non-binary collation") {
+ val relation = LocalRelation(AttributeReference("a",
StringType("UTF8_LCASE"))())
+ val lcase = StringType("UTF8_LCASE")
+ val originalQuery =
+ relation.where(Like(relation.output.head, Literal.create("a%b%", lcase),
'\\')).analyze
+ comparePlans(Optimize.execute(originalQuery), originalQuery)
+ }
+
+ test("prefix guard derivation is idempotent") {
Review Comment:
`Optimize` here is `Once`. Production runs `LikeSimplification` in the
fixed-point Operator Optimization batch with `BooleanSimplification` /
`PruneFilters` / etc., which can copy the residual `Like` and drop the tag.
Could we run this through that batch (or the real `Optimizer`) so a lost tag
would fail rather than silently re-wrap?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
