This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 666afb88e670 [SPARK-57932][4.2][SQL] Fix regexp_instr and
regexp_replace returning wrong results for supplementary characters
666afb88e670 is described below
commit 666afb88e670db245889a3aaa474c08d4a0f52e1
Author: Kousuke Saruta <[email protected]>
AuthorDate: Tue Jul 14 08:41:32 2026 +0900
[SPARK-57932][4.2][SQL] Fix regexp_instr and regexp_replace returning wrong
results for supplementary characters
### What changes were proposed in this pull request?
This PR backports SPARK-57932 (#56998) for `branch-4.2` with conflict
resolution.
`regexp_instr` returned matcher.start() + 1 (a UTF-16 code-unit index), and
`regexp_replace`'s position argument was used as a UTF-16 offset in
matcher.region(pos-1, ...) and its out-of-range guard. Both now use code-point
positions via codePointCount/offsetByCodePoints.
### Why are the changes needed?
Spark string positions are 1-based code points (instr, locate, position,
substring, length). For strings containing supplementary characters (code
points > U+FFFF, e.g. emojis), these two functions gave wrong results:
- regexp_instr('😀ab', 'ab') returned 3 instead of 2.
- regexp_replace('😀aXa', 'a', 'Z', 3) returned 😀ZXZ instead of 😀aXZ
(replaced a match before position); regexp_replace('😀😀', '😀', 'Z', 3) returned
😀Z instead of the input unchanged. This is the sibling of the LIKE fix in
SPARK-55453.
### Does this PR introduce _any_ user-facing change?
Yes. regexp_instr and regexp_replace now return code-point-consistent
results for strings with supplementary characters.
### How was this patch tested?
Added UT.
### Was this patch authored or co-authored using generative AI tooling?
Yes.
Closes #57213 from sarutak/SPARK-57932-branch-4.2.
Lead-authored-by: Kousuke Saruta <[email protected]>
Co-authored-by: Eric Yang <[email protected]>
Signed-off-by: Kousuke Saruta <[email protected]>
---
.../catalyst/expressions/regexpExpressions.scala | 17 +++++++-----
.../expressions/RegexpExpressionsSuite.scala | 31 ++++++++++++++++++++++
2 files changed, 41 insertions(+), 7 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index 5ad360a54e8d..6740db92b6c8 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -740,9 +740,9 @@ case class RegExpReplace(subject: Expression, regexp:
Expression, rep: Expressio
}
val source = s.toString()
val position = i.asInstanceOf[Int] - 1
- if (position == 0 || position < source.length) {
+ if (position == 0 || position < source.codePointCount(0, source.length)) {
val m = pattern.matcher(source)
- m.region(position, source.length)
+ m.region(source.offsetByCodePoints(0, position), source.length)
result.delete(0, result.length())
while (m.find) {
try {
@@ -795,9 +795,9 @@ case class RegExpReplace(subject: Expression, regexp:
Expression, rep: Expressio
}
String $source = $subject.toString();
int $position = $pos - 1;
- if ($position == 0 || $position < $source.length()) {
+ if ($position == 0 || $position < $source.codePointCount(0,
$source.length())) {
$classNameStringBuilder $termResult = new $classNameStringBuilder();
- $matcher.region($position, $source.length());
+ $matcher.region($source.offsetByCodePoints(0, $position),
$source.length());
while ($matcher.find()) {
try {
@@ -1196,9 +1196,10 @@ case class RegExpInStr(subject: Expression, regexp:
Expression, idx: Expression)
override def nullSafeEval(s: Any, r: Any, i: Any): Any = {
try {
- val m = getLastMatcher(s, r)
+ val source = s.toString
+ val m = getLastMatcher(source, r)
if (m.find) {
- m.toMatchResult.start() + 1
+ source.codePointCount(0, m.toMatchResult.start()) + 1
} else {
0
}
@@ -1212,6 +1213,7 @@ case class RegExpInStr(subject: Expression, regexp:
Expression, idx: Expression)
override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode = {
val matcher = ctx.freshName("matcher")
+ val source = ctx.freshName("source")
val setEvNotNull = if (nullable) {
s"${ev.isNull} = false;"
} else {
@@ -1225,7 +1227,8 @@ case class RegExpInStr(subject: Expression, regexp:
Expression, idx: Expression)
| ${RegExpUtils.initLastMatcherCode(ctx, subject, regexp, matcher,
prettyName,
collationId)}
| if ($matcher.find()) {
- | ${ev.value} = $matcher.toMatchResult().start() + 1;
+ | String $source = $subject.toString();
+ | ${ev.value} = $source.codePointCount(0,
$matcher.toMatchResult().start()) + 1;
| } else {
| ${ev.value} = 0;
| }
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
index 9b5a8f2fce06..270480a63c0f 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
@@ -366,6 +366,26 @@ class RegexpExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
RegExpReplace(Literal("\"quote"), Literal("\"quote"),
Literal("\"quote")) :: Nil)
}
+ test("SPARK-57932: regexp_replace position is a code-point position for
supplementary chars") {
+ // scalastyle:off nonascii
+ checkEvaluation(
+ RegExpReplace(Literal("😀aXa"), Literal("a"), Literal("Z"), Literal(3)),
+ "😀aXZ")
+ checkEvaluation(
+ RegExpReplace(Literal("😀😀"), Literal("😀"),
+ Literal("Z"), Literal(3)),
+ "😀😀")
+ // Position beyond the string length: nothing is replaced.
+ checkEvaluation(
+ RegExpReplace(Literal("😀a"), Literal("a"), Literal("Z"), Literal(4)),
+ "😀a")
+ // Position at the last code-point.
+ checkEvaluation(
+ RegExpReplace(Literal("a😀b"), Literal("b"), Literal("Z"), Literal(3)),
+ "a😀Z")
+ // scalastyle:on nonascii
+ }
+
test("SPARK-22570: RegExpReplace should not create a lot of global
variables") {
val ctx = new CodegenContext
RegExpReplace(Literal("100"), Literal("(\\d+)"),
Literal("num")).genCode(ctx)
@@ -627,6 +647,17 @@ class RegexpExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
new RegExpInStr(Literal("\"quote"), Literal("\"quote")) :: Nil)
}
+ test("SPARK-57932: regexp_instr returns a code-point position for
supplementary characters") {
+ // scalastyle:off nonascii
+ checkEvaluation(RegExpInStr(Literal("😀ab"), Literal("ab"), Literal(0)), 2)
+ checkEvaluation(RegExpInStr(Literal("a😀b"), Literal("b"), Literal(0)), 3)
+ checkEvaluation(
+ RegExpInStr(Literal("😀😁xy"), Literal("xy"), Literal(0)), 3)
+ // A match made up entirely of supplementary characters.
+ checkEvaluation(RegExpInStr(Literal("😀😁😂"), Literal("😁"), Literal(0)), 2)
+ // scalastyle:on nonascii
+ }
+
test("SPARK-39758: invalid regexp pattern") {
val s = $"s".string.at(0)
val p = $"p".string.at(1)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
