This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.3 by this push:
new f4473720aceb [SPARK-48973][SQL] Fix mask to handle characters outside
the BMP
f4473720aceb is described below
commit f4473720aceb853db1c1304f3af48b6ca0698ec5
Author: Eric Yang <[email protected]>
AuthorDate: Mon Aug 10 10:55:43 2026 +0800
[SPARK-48973][SQL] Fix mask to handle characters outside the BMP
### What changes were proposed in this pull request?
Make `Mask.transformInput` iterate by code point instead of by UTF-16 code
unit, and read the replacement character with `codePointAt(0)` instead of
`charAt(0)`. The character classification logic is unchanged; codegen needs no
change because `doGenCode` calls the same static method. Also hoists the
replacement lookup out of the per-character loop, which allocated a String per
character previously.
### Why are the changes needed?
`mask` is documented for "creating copies of tables with sensitive
information removed", but sees each supplementary character as two unpaired
surrogates:
```SQL
SELECT mask('A𝐀 1𝟎'); -- 'X𝐀 n𝟎', expected 'XX nn'
SELECT mask('🙂', 'Y', 'y', 'n', '*'); -- '**', expected '*'
SELECT mask('ABC', '🙂'); -- '???', expected '🙂🙂🙂'
```
Both halves classify as SURROGATE, so the first silently skips redaction:
supplementary letters and digits (U+1D400, or Adlam at U+1E900) are
UPPERCASE_LETTER / LOWERCASE_LETTER / DECIMAL_DIGIT_NUMBER and should be
masked, but take the otherChar branch and pass through. The second replaces one
input character twice. The third is internally inconsistent:
checkInputDataTypes accepts '🙂' as one character (UTF8String.numChars counts
code points), then evaluation truncates it.
The char-wise iteration came from Hive's GenericUDFMask, but Spark did not
port Hive's argument handling, and Spark is code-point based elsewhere (length,
substring, instr, locate, position) - with the same fix applied (SPARK-55453,
SPARK-57932, etc). Only the iteration changes here; classification still
matches Hive, so OTHER_LETTER scripts (CJK, Hebrew, ...) keep taking the
otherChar branch as today.
### Does this PR introduce _any_ user-facing change?
Yes. mask now treats a supplementary character as a single character. Input
consisting only of BMP characters is unaffected. Supplementary letters and
digits are now masked where they were previously passed through, the result
keeps the input's character count, and a supplementary replacement character is
used as given instead of becoming '?'.
### How was this patch tested?
New test cases added
### Was this patch authored or co-authored using generative AI tooling?
Yes.
Closes #57858 from jiwen624/SPARK-48973.
Authored-by: Eric Yang <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit c9aeac253b78ee910134fbd9191944f6021e4ae1)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../sql/catalyst/expressions/maskExpressions.scala | 49 +++++++++++++++-------
.../expressions/StringExpressionsSuite.scala | 27 ++++++++++++
2 files changed, 61 insertions(+), 15 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala
index 9613bc25d65f..a283eb1c8143 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala
@@ -319,6 +319,7 @@ object Mask {
val MASKED_DIGIT = 'n'
// This value helps to retain original value in the input by ignoring the
replacement rules
val MASKED_IGNORE = null
+ private val IGNORE_CODE_POINT = -1
def transformInput(
input: Any,
@@ -330,29 +331,47 @@ object Mask {
val transformedString = if (input == null) {
null
} else {
- input.toString.map {
- transformChar(_, maskUpper, maskLower, maskDigit, maskOther).toChar
+ val upper = replacementCodePoint(maskUpper)
+ val lower = replacementCodePoint(maskLower)
+ val digit = replacementCodePoint(maskDigit)
+ val other = replacementCodePoint(maskOther)
+ val str = input.toString
+ val sb = new java.lang.StringBuilder(str.length)
+ var i = 0
+ while (i < str.length) {
+ val codePoint = str.codePointAt(i)
+ sb.appendCodePoint(transformCodePoint(codePoint, upper, lower, digit,
other))
+ i += Character.charCount(codePoint)
}
+ sb.toString
}
org.apache.spark.unsafe.types.UTF8String.fromString(transformedString)
}
- private def transformChar(
- c: Char,
- maskUpper: Any,
- maskLower: Any,
- maskDigit: Any,
- maskOther: Any): Int = {
+ private def replacementCodePoint(option: Any): Int = {
+ if (option != MASKED_IGNORE) {
+ option.asInstanceOf[UTF8String].toString.codePointAt(0)
+ } else {
+ IGNORE_CODE_POINT
+ }
+ }
+
+ private def transformCodePoint(
+ codePoint: Int,
+ maskUpper: Int,
+ maskLower: Int,
+ maskDigit: Int,
+ maskOther: Int): Int = {
- def maskedChar(c: Char, option: Any): Char = {
- if (option != MASKED_IGNORE)
option.asInstanceOf[UTF8String].toString.charAt(0) else c
+ def maskedCodePoint(replacement: Int): Int = {
+ if (replacement != IGNORE_CODE_POINT) replacement else codePoint
}
- Character.getType(c) match {
- case Character.UPPERCASE_LETTER => maskedChar(c, maskUpper)
- case Character.LOWERCASE_LETTER => maskedChar(c, maskLower)
- case Character.DECIMAL_DIGIT_NUMBER => maskedChar(c, maskDigit)
- case _ => maskedChar(c, maskOther)
+ Character.getType(codePoint) match {
+ case Character.UPPERCASE_LETTER => maskedCodePoint(maskUpper)
+ case Character.LOWERCASE_LETTER => maskedCodePoint(maskLower)
+ case Character.DECIMAL_DIGIT_NUMBER => maskedCodePoint(maskDigit)
+ case _ => maskedCodePoint(maskOther)
}
}
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index d2a58b95b83e..4ae723f33b39 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -451,6 +451,33 @@ class StringExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
}
+ test("SPARK-48973: Mask with supplementary characters") {
+ def cp(codePoint: Int): String = new String(Character.toChars(codePoint))
+ val smile = cp(0x1F642)
+ val boldA = cp(0x1D400)
+ val boldSmallA = cp(0x1D41A)
+ val boldZero = cp(0x1D7CE)
+
+ checkEvaluation(
+ new Mask(Literal(smile), Literal('Y'), Literal('y'), Literal('n'),
Literal('*')), "*")
+ checkEvaluation(new Mask(Literal("ABC"), Literal(smile)), smile * 3)
+ checkEvaluation(new Mask(Literal(s"A$boldA 1$boldZero")), "XX nn")
+ // Supplementary upper-case, lower-case and digit characters are each
classified and
+ // replaced like their BMP counterparts.
+ checkEvaluation(new Mask(Literal(s"$boldA$boldSmallA$boldZero")), "Xxn")
+ // A supplementary replacement applied to a supplementary input.
+ checkEvaluation(new Mask(Literal(boldSmallA), Literal('Y'),
Literal(smile)), smile)
+
+ // A supplementary character must round-trip intact through the retain
path, both when it
+ // falls into the otherChar category and when its own category is set to
retain.
+ checkEvaluation(new Mask(Literal(smile)), smile)
+ checkEvaluation(new Mask(Literal(s"a${smile}1")), s"x${smile}n")
+ checkEvaluation(new Mask(Literal(boldA), Literal(null, StringType)), boldA)
+ checkEvaluation(
+ new Mask(Literal(boldZero), Literal('Y'), Literal('y'), Literal(null,
StringType)),
+ boldZero)
+ }
+
test("SPARK-42384: Mask with null input") {
val NULL_LITERAL = Literal(null, StringType)
checkEvaluation(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
