This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new d9a289e35542 [SPARK-57253][SQL] Add `jaro_winkler_similarity` built-in
function
d9a289e35542 is described below
commit d9a289e355427b4e4c510fd4bd1170abac90572d
Author: Kousuke Saruta <[email protected]>
AuthorDate: Mon Jun 8 20:53:44 2026 -0700
[SPARK-57253][SQL] Add `jaro_winkler_similarity` built-in function
### What changes were proposed in this pull request?
This PR adds a new built-in string function `jaro_winkler_similarity(str1,
str2)` that computes the Jaro-Winkler similarit
y between two strings. The result is a double between 0.0 (no similarity)
and 1.0 (identical strings).
The Jaro-Winkler metric is an extension of the Jaro similarity that gives a
bonus for common prefixes (up to 4 characters)
, making it especially suited for short strings such as names.
Changes:
* `ExpressionImplUtils.jaroWinklerSimilarity()` — core algorithm
implementation
* `JaroWinkler` expression as `RuntimeReplaceable` + `StaticInvoke`
* Registration as `jaro_winkler_similarity` in `FunctionRegistry`
* Scala DataFrame API in `functions.scala`
* PySpark API in `pyspark.sql.functions` + Spark Connect
* Unit tests covering basic cases, symmetry, multi-byte strings, and null
handling
### Why are the changes needed?
Jaro-Winkler is one of the most commonly used string similarity metrics for
record linkage, deduplication, and fuzzy matching. It is available as a
built-in function in DuckDB, SQL Server 2025, and various other engines, but
Spark users currently need to implement it as a UDF.
A built-in function provides:
- Codegen support (no UDF overhead in whole-stage codegen pipelines)
- Consistency with the existing `levenshtein` function
- Better discoverability for SQL users
### Does this PR introduce _any_ user-facing change?
Yes. A new SQL function `jaro_winkler_similarity` is added.
### How was this patch tested?
Added new tests.
### Was this patch authored or co-authored using generative AI tooling?
Kiro CLI / Claude
Closes #56310 from sarutak/jaro-winkler.
Authored-by: Kousuke Saruta <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit b67073f19ecd5a89193f96926cd832bb86a50382)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../source/reference/pyspark.sql/functions.rst | 1 +
python/pyspark/sql/connect/functions/builtin.py | 7 +++
python/pyspark/sql/functions/__init__.py | 1 +
python/pyspark/sql/functions/builtin.py | 34 ++++++++++++
python/pyspark/sql/tests/test_functions.py | 8 +++
.../scala/org/apache/spark/sql/functions.scala | 9 ++++
.../catalyst/expressions/ExpressionImplUtils.java | 61 ++++++++++++++++++++++
.../sql/catalyst/analysis/FunctionRegistry.scala | 1 +
.../catalyst/expressions/stringExpressions.scala | 36 +++++++++++++
.../expressions/StringExpressionsSuite.scala | 38 ++++++++++++++
.../sql-functions/sql-expression-schema.md | 1 +
.../apache/spark/sql/StringFunctionsSuite.scala | 13 +++++
12 files changed, 210 insertions(+)
diff --git a/python/docs/source/reference/pyspark.sql/functions.rst
b/python/docs/source/reference/pyspark.sql/functions.rst
index 4d91d15a87d0..4b9084547b9a 100644
--- a/python/docs/source/reference/pyspark.sql/functions.rst
+++ b/python/docs/source/reference/pyspark.sql/functions.rst
@@ -180,6 +180,7 @@ String Functions
initcap
instr
is_valid_utf8
+ jaro_winkler_similarity
lcase
left
length
diff --git a/python/pyspark/sql/connect/functions/builtin.py
b/python/pyspark/sql/connect/functions/builtin.py
index 7c7edef02f0d..11ec77c7d30c 100644
--- a/python/pyspark/sql/connect/functions/builtin.py
+++ b/python/pyspark/sql/connect/functions/builtin.py
@@ -2704,6 +2704,13 @@ def levenshtein(
levenshtein.__doc__ = pysparkfuncs.levenshtein.__doc__
+def jaro_winkler_similarity(left: "ColumnOrName", right: "ColumnOrName") ->
Column:
+ return _invoke_function_over_columns("jaro_winkler_similarity", left,
right)
+
+
+jaro_winkler_similarity.__doc__ = pysparkfuncs.jaro_winkler_similarity.__doc__
+
+
def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column:
return _invoke_function("locate", lit(substr), _to_col(str), lit(pos))
diff --git a/python/pyspark/sql/functions/__init__.py
b/python/pyspark/sql/functions/__init__.py
index 9a9b18a29278..3801ba0e51e5 100644
--- a/python/pyspark/sql/functions/__init__.py
+++ b/python/pyspark/sql/functions/__init__.py
@@ -141,6 +141,7 @@ __all__ = [ # noqa: F405
"initcap",
"instr",
"is_valid_utf8",
+ "jaro_winkler_similarity",
"lcase",
"left",
"length",
diff --git a/python/pyspark/sql/functions/builtin.py
b/python/pyspark/sql/functions/builtin.py
index d0dfb2278565..4c11bbca4152 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -15577,6 +15577,40 @@ def levenshtein(
)
+@_try_remote_functions
+def jaro_winkler_similarity(left: "ColumnOrName", right: "ColumnOrName") ->
Column:
+ """Computes the Jaro-Winkler similarity between the two given strings.
+
+ The result is a double between 0.0 (no similarity) and 1.0 (identical
strings).
+
+ .. versionadded:: 4.3.0
+
+ Parameters
+ ----------
+ left : :class:`~pyspark.sql.Column` or column name
+ first column value.
+ right : :class:`~pyspark.sql.Column` or column name
+ second column value.
+
+ Returns
+ -------
+ :class:`~pyspark.sql.Column`
+ Jaro-Winkler similarity as a double value.
+
+ Examples
+ --------
+ >>> from pyspark.sql import functions as sf
+ >>> df = spark.createDataFrame([('MARTHA', 'MARHTA')], ['l', 'r'])
+ >>> df.select(sf.jaro_winkler_similarity('l', 'r')).show()
+ +-----------------------------+
+ |jaro_winkler_similarity(l, r)|
+ +-----------------------------+
+ | 0.9611111111111111|
+ +-----------------------------+
+ """
+ return _invoke_function_over_columns("jaro_winkler_similarity", left,
right)
+
+
@_try_remote_functions
def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column:
"""
diff --git a/python/pyspark/sql/tests/test_functions.py
b/python/pyspark/sql/tests/test_functions.py
index f0484cb98c83..33579f7066e5 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -1014,6 +1014,14 @@ class FunctionsTestsMixin:
self.assertAlmostEqual(sum_result[0], 4.0, places=4)
self.assertAlmostEqual(sum_result[1], 6.0, places=4)
+ def test_jaro_winkler_similarity_function(self):
+ df = self.spark.createDataFrame([("MARTHA", "MARHTA")], ["l", "r"])
+ result = df.select(F.jaro_winkler_similarity(df.l, df.r)).first()[0]
+ self.assertAlmostEqual(result, 0.9611111111111111, places=10)
+ # Null handling
+ null_result = df.select(F.jaro_winkler_similarity(df.l,
F.lit(None))).first()[0]
+ self.assertIsNone(null_result)
+
def test_between_function(self):
df = self.spark.createDataFrame(
[Row(a=1, b=2, c=3), Row(a=2, b=1, c=3), Row(a=4, b=1, c=4)]
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
index 115e2c572753..ab21a784796d 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
@@ -5220,6 +5220,15 @@ object functions {
*/
def levenshtein(l: Column, r: Column): Column = Column.fn("levenshtein", l,
r)
+ /**
+ * Computes the Jaro-Winkler similarity between the two given string
columns. The result is a
+ * double between 0.0 (no similarity) and 1.0 (identical).
+ * @group string_funcs
+ * @since 4.3.0
+ */
+ def jaro_winkler_similarity(l: Column, r: Column): Column =
+ Column.fn("jaro_winkler_similarity", l, r)
+
/**
* Locate the position of the first occurrence of substr.
*
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java
index d44922261674..e744ae3e2fc2 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java
@@ -467,4 +467,65 @@ public class ExpressionImplUtils {
}
numberFormat.applyLocalizedPattern(pattern.toString());
}
+
+ /**
+ * Returns the Jaro-Winkler similarity between two strings.
+ * The result is a double between 0 and 1, where 1 means identical.
+ *
+ * Note: This implementation uses String.charAt() which operates on UTF-16
code units.
+ * Strings containing supplementary characters (surrogate pairs) may produce
+ * inaccurate results.
+ */
+ public static double jaroWinklerSimilarity(UTF8String left, UTF8String
right) {
+ String s1 = left.toString();
+ String s2 = right.toString();
+
+ int len1 = s1.length();
+ int len2 = s2.length();
+
+ if (len1 == 0 && len2 == 0) return 1.0;
+ if (len1 == 0 || len2 == 0) return 0.0;
+
+ int matchWindow = Math.max(0, Math.max(len1, len2) / 2 - 1);
+
+ boolean[] s1Matched = new boolean[len1];
+ boolean[] s2Matched = new boolean[len2];
+
+ int matches = 0;
+ int transpositions = 0;
+
+ for (int i = 0; i < len1; i++) {
+ int start = Math.max(0, i - matchWindow);
+ int end = Math.min(i + matchWindow + 1, len2);
+ for (int j = start; j < end; j++) {
+ if (s2Matched[j] || s1.charAt(i) != s2.charAt(j)) continue;
+ s1Matched[i] = true;
+ s2Matched[j] = true;
+ matches++;
+ break;
+ }
+ }
+
+ if (matches == 0) return 0.0;
+
+ int k = 0;
+ for (int i = 0; i < len1; i++) {
+ if (!s1Matched[i]) continue;
+ while (!s2Matched[k]) k++;
+ if (s1.charAt(i) != s2.charAt(k)) transpositions++;
+ k++;
+ }
+
+ double jaro = ((double) matches / len1
+ + (double) matches / len2
+ + (double) (matches - transpositions / 2.0) / matches) / 3.0;
+
+ int prefix = 0;
+ for (int i = 0; i < Math.min(4, Math.min(len1, len2)); i++) {
+ if (s1.charAt(i) == s2.charAt(i)) prefix++;
+ else break;
+ }
+
+ return jaro + prefix * 0.1 * (1.0 - jaro);
+ }
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index ac1fbb9f9bef..1f216ad6a033 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -630,6 +630,7 @@ object FunctionRegistry {
expression[Length]("length"),
expression[Length]("len", setAlias = true, Some("3.4.0")),
expression[Levenshtein]("levenshtein"),
+ expression[JaroWinkler]("jaro_winkler_similarity"),
expression[Luhncheck]("luhn_check"),
expression[Like]("like"),
expression[ILike]("ilike"),
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 45ec0e6a8b30..ec04f9f713fc 100755
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -2721,6 +2721,42 @@ case class Levenshtein(
}
}
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = "_FUNC_(str1, str2) - Returns the Jaro-Winkler similarity between
the two given strings. The result is a double between 0 and 1, where 1 means
identical.",
+ examples = """
+ Examples:
+ > SELECT _FUNC_('MARTHA', 'MARHTA');
+ 0.9611111111111111
+ > SELECT _FUNC_('kitten', 'sitting');
+ 0.746031746031746
+ > SELECT _FUNC_('ABC', 'XYZ');
+ 0.0
+ """,
+ since = "4.3.0",
+ group = "string_funcs")
+// scalastyle:on line.size.limit
+case class JaroWinkler(left: Expression, right: Expression)
+ extends RuntimeReplaceable with ImplicitCastInputTypes with
BinaryLike[Expression] {
+
+ override lazy val replacement: Expression = StaticInvoke(
+ classOf[ExpressionImplUtils],
+ DoubleType,
+ "jaroWinklerSimilarity",
+ Seq(left, right),
+ inputTypes)
+
+ override def inputTypes: Seq[AbstractDataType] = Seq(
+ StringTypeWithCollation(supportsTrimCollation = true),
+ StringTypeWithCollation(supportsTrimCollation = true))
+
+ override def prettyName: String = "jaro_winkler_similarity"
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression, newRight: Expression): Expression =
+ copy(left = newLeft, right = newRight)
+}
+
/**
* A function that return Soundex code of the given string expression.
*/
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index aac4fafb7802..19df4e5f39e9 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -662,6 +662,44 @@ class StringExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
GenerateUnsafeProjection.generate(Levenshtein(Literal("\"quotea"),
Literal("\"quoteb")) :: Nil)
}
+ test("Jaro-Winkler similarity") {
+ // Identical strings
+ checkEvaluation(JaroWinkler(Literal("ABC"), Literal("ABC")), 1.0)
+ // Completely different
+ checkEvaluation(JaroWinkler(Literal("ABC"), Literal("XYZ")), 0.0)
+ // Empty strings
+ checkEvaluation(JaroWinkler(Literal(""), Literal("")), 1.0)
+ checkEvaluation(JaroWinkler(Literal("ABC"), Literal("")), 0.0)
+ checkEvaluation(JaroWinkler(Literal(""), Literal("ABC")), 0.0)
+ // Classic example: MARTHA vs MARHTA (well-known reference value)
+ checkEvaluation(JaroWinkler(Literal("MARTHA"), Literal("MARHTA")),
0.9611111111111111)
+ // Another well-known example
+ checkEvaluation(JaroWinkler(Literal("DWAYNE"), Literal("DUANE")),
0.8400000000000001)
+ // Symmetry: jaro_winkler(a, b) == jaro_winkler(b, a)
+ checkEvaluation(JaroWinkler(Literal("MARHTA"), Literal("MARTHA")),
0.9611111111111111)
+ checkEvaluation(JaroWinkler(Literal("sitting"), Literal("kitten")),
+ JaroWinkler(Literal("kitten"), Literal("sitting")).eval(null))
+ // Strings with very different lengths
+ assert(JaroWinkler(Literal("a"), Literal("abcdefgh")).eval(null)
+ .asInstanceOf[Double] > 0.0)
+ checkEvaluation(JaroWinkler(Literal("abc"), Literal("abcdefgh")),
+ JaroWinkler(Literal("abcdefgh"), Literal("abc")).eval(null))
+ // Single character strings
+ checkEvaluation(JaroWinkler(Literal("a"), Literal("a")), 1.0)
+ checkEvaluation(JaroWinkler(Literal("a"), Literal("b")), 0.0)
+ // Multi-byte (Japanese) strings
+ // scalastyle:off nonascii
+ checkEvaluation(JaroWinkler(Literal("東京都"), Literal("東京都")), 1.0)
+ checkEvaluation(JaroWinkler(Literal("東京都"), Literal("東京府")),
+ JaroWinkler(Literal("東京府"), Literal("東京都")).eval(null))
+ assert(JaroWinkler(Literal("東京都"), Literal("東京府")).eval(null)
+ .asInstanceOf[Double] > 0.8)
+ // scalastyle:on nonascii
+ // Null handling
+ checkEvaluation(JaroWinkler(Literal.create(null, StringType),
Literal("ABC")), null)
+ checkEvaluation(JaroWinkler(Literal("ABC"), Literal.create(null,
StringType)), null)
+ }
+
test("soundex unit test") {
checkEvaluation(SoundEx(Literal("ZIN")), "Z500")
checkEvaluation(SoundEx(Literal("SU")), "S000")
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 9ac6e49b0343..be8559d16cb7 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -181,6 +181,7 @@
| org.apache.spark.sql.catalyst.expressions.IsNotNull | isnotnull | SELECT
isnotnull(1) | struct<(1 IS NOT NULL):boolean> |
| org.apache.spark.sql.catalyst.expressions.IsNull | isnull | SELECT isnull(1)
| struct<(1 IS NULL):boolean> |
| org.apache.spark.sql.catalyst.expressions.IsValidUTF8 | is_valid_utf8 |
SELECT is_valid_utf8('Spark') | struct<is_valid_utf8(Spark):boolean> |
+| org.apache.spark.sql.catalyst.expressions.JaroWinkler |
jaro_winkler_similarity | SELECT jaro_winkler_similarity('MARTHA', 'MARHTA') |
struct<jaro_winkler_similarity(MARTHA, MARHTA):double> |
| org.apache.spark.sql.catalyst.expressions.JsonObjectKeys | json_object_keys
| SELECT json_object_keys('{}') | struct<json_object_keys({}):array<string>> |
| org.apache.spark.sql.catalyst.expressions.JsonToStructs | from_json | SELECT
from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE') | struct<from_json({"a":1,
"b":0.8}):struct<a:int,b:double>> |
| org.apache.spark.sql.catalyst.expressions.JsonTuple | json_tuple | SELECT
json_tuple('{"a":1, "b":2}', 'a', 'b') | struct<c0:string,c1:string> |
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
index b507cb62f9d1..772773cb07c8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
@@ -145,6 +145,19 @@ class StringFunctionsSuite extends SharedSparkSession {
checkAnswer(df.selectExpr("levenshtein(l, null, 0)"), Seq(Row(null),
Row(null)))
}
+ test("string jaro_winkler_similarity") {
+ val df = Seq(("MARTHA", "MARHTA"), ("ABC", "XYZ")).toDF("l", "r")
+ checkAnswer(
+ df.select(jaro_winkler_similarity($"l", $"r")),
+ Seq(Row(0.9611111111111111), Row(0.0)))
+ checkAnswer(
+ df.selectExpr("jaro_winkler_similarity(l, r)"),
+ Seq(Row(0.9611111111111111), Row(0.0)))
+ checkAnswer(
+ df.select(jaro_winkler_similarity($"l", lit(null))),
+ Seq(Row(null), Row(null)))
+ }
+
test("string rlike / regexp / regexp_like") {
val df = Seq(
("1a 2b 14m", "\\d+b"),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
