This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new dc73d212ae62 [SPARK-58628][SQL] Transpiler: skip null guards for
is-not-None-narrowed parameters
dc73d212ae62 is described below
commit dc73d212ae6213f0079ffd01dce7dad99e7db827
Author: jzhan-2026 <[email protected]>
AuthorDate: Fri Oct 30 17:05:38 2026 -0700
[SPARK-58628][SQL] Transpiler: skip null guards for is-not-None-narrowed
parameters
Three related improvements to the Python UDF transpiler:
1. **Null narrowing.** When a parameter is guarded by `x is not None` (or
`x is None`) in an `if`/ternary test, the transpiler tracks that the parameter
is proven non-null in the appropriate branch. Ordering comparisons (`<`, `<=`,
`>`, `>=`) and `len()` calls on such parameters skip the `raise_error` null
guard entirely.
2. **2-statement early-return normalization.** A common guard idiom:
```python
if x is None:
return default
return x > 0
is normalized into a single if/else before lowering, so null narrowing
applies to the main body.
3. UserWarning on null guard emission. When a null guard is still emitted
(non-null cannot be proven), the transpiler warns at UDF construction time.
Co-authored with jing.zhansnowflake.com
Any ordering comparison in a transpiled UDF previously always emitted a
raise_error guard, even when the code already checked x is not None. This
produced unnecessary plan nodes and blocked clean transpilation of the most
common Python null-guard idiom.
Yes. UDFs with is not None-guarded comparisons produce leaner plans (no
raise_error node in the guarded branch). The UserWarning at construction time
is new when unguarded comparisons remain. Behavior on non-null inputs is
unchanged.
Four new test cases in test_udf_transpile_unit.py:
- test_udf_transpile_null_narrowing_skips_null_guard — if/ternary with is
not None guard: plan has no raise_error, results match Python
- test_udf_transpile_early_return_null_guard — 2-stmt early-return pattern
transpiles with no raise_error
- test_udf_transpile_null_guard_warning — unguarded comparison emits
UserWarning
- test_udf_transpile_null_narrowing_len — guarded len() has no raise_error
in plan
Co-authored by: Claude Sonnet 4.6
Closes #59108 from jzhan-2026/spark-58628-non-nullable-null-guard.
Authored-by: jzhan-2026 <[email protected]>
Co-authored-by: [email protected] <[email protected]>
Signed-off-by: Holden Karau <[email protected]>
---
.../pyspark/sql/tests/test_udf_transpile_unit.py | 144 ++++++++++++++++++++-
python/pyspark/sql/transpile.py | 121 +++++++++++++++--
2 files changed, 254 insertions(+), 11 deletions(-)
diff --git a/python/pyspark/sql/tests/test_udf_transpile_unit.py
b/python/pyspark/sql/tests/test_udf_transpile_unit.py
index 5c0309bb53b3..c19c38814fff 100644
--- a/python/pyspark/sql/tests/test_udf_transpile_unit.py
+++ b/python/pyspark/sql/tests/test_udf_transpile_unit.py
@@ -246,7 +246,7 @@ class UDFTranspileUnitTests(ReusedSQLTestCase):
if x is not None:
return x << 1
- def multi_statement(x): # > 1 top-level statement, not handled.
+ def multi_statement(x): # First stmt is an assignment, not an
if-guard.
y = 1
return x + y if x is not None else 0
@@ -2750,6 +2750,148 @@ class UDFTranspileUnitTests(ReusedSQLTestCase):
self.assertEqual(len(combos_u), 3)
self.assertEqual({c[0] for c in combos_u}, {"integer", "float",
"string"})
+ def test_udf_transpile_null_narrowing_skips_null_guard(self):
+ # When a parameter is guarded by `x is not None` in an
+ # if/ternary test, the transpiler should skip the raise_error null
guard
+ # for ordering comparisons on that parameter within the guarded branch.
+ def if_guard_gt(x):
+ return (x > 0) if x is not None else None
+
+ def if_guard_and_lt(a, b):
+ return (a < b) if a is not None and b is not None else None
+
+ def ternary_guard_ge(x):
+ return (x >= 0) if x is not None else None
+
+ def is_none_ternary(x):
+ return None if x is None else x > 0
+
+ cases_if = [
+ # if/else with `is not None` guard
+ ("if_guard_gt", if_guard_gt),
+ # if/else with `is not None` and-chain (two params)
+ ("if_guard_and_lt", if_guard_and_lt),
+ # ternary with `is not None` guard
+ ("ternary_guard_ge", ternary_guard_ge),
+ # `is None` guard: non-null in the else branch
+ ("is_none_ternary", is_none_ternary),
+ ]
+ with self.sql_conf(_TRANSPILE_ON):
+ for name, func in cases_if:
+ with self.subTest(func=name):
+ u, warnings_text = self._udf_and_warnings(func,
BooleanType())
+ self.assertTrue(u.transpiled, f"{name} did not transpile:
{warnings_text}")
+ arity = func.__code__.co_argcount
+ if arity == 1:
+ df = self.spark.createDataFrame([(1,), (-1,),
(None,)], "a long")
+ plan = self._optimized_plan(df.select(u("a")))
+ self.assertNotIn("raise_error", plan, f"{name}: plan
had raise_error")
+ self.assertEqual(
+ [True, False, None],
+ [r[0] for r in df.select(u("a")).collect()],
+ name,
+ )
+ else:
+ df = self.spark.createDataFrame(
+ [(1, 2), (2, 1), (None, 1)], "a long, b long"
+ )
+ plan = self._optimized_plan(df.select(u("a", "b")))
+ self.assertNotIn("raise_error", plan, f"{name}: plan
had raise_error")
+ self.assertEqual(
+ [True, False, None],
+ [r[0] for r in df.select(u("a", "b")).collect()],
+ name,
+ )
+
+ def test_udf_transpile_early_return_null_guard(self):
+ # A 2-statement early-return pattern
+ # `if x is None: return y\nreturn body` is normalized to a single
+ # if/else so null narrowing applies to the main body.
+ def early_return_gt(x):
+ if x is None:
+ return None
+ return x > 0
+
+ def early_return_lt_default(x):
+ if x is None:
+ return False
+ return x < 10
+
+ with self.sql_conf(_TRANSPILE_ON):
+ for name, func, rt, rows, expected in [
+ (
+ "early_return_gt",
+ early_return_gt,
+ BooleanType(),
+ [(1,), (-1,), (None,)],
+ [True, False, None],
+ ),
+ (
+ "early_return_lt_default",
+ early_return_lt_default,
+ BooleanType(),
+ [(5,), (15,), (None,)],
+ [True, False, False],
+ ),
+ ]:
+ with self.subTest(func=name):
+ u, warnings_text = self._udf_and_warnings(func, rt)
+ self.assertTrue(u.transpiled, f"{name} did not transpile:
{warnings_text}")
+ df = self.spark.createDataFrame(rows, "a long")
+ plan = self._optimized_plan(df.select(u("a")))
+ self.assertNotIn("raise_error", plan, f"{name}: plan had
raise_error")
+ self.assertEqual(
+ expected,
+ [r[0] for r in df.select(u("a")).collect()],
+ name,
+ )
+
+ def test_udf_transpile_null_guard_warning(self):
+ # When the transpiler still emits a null guard (because
+ # non-null cannot be proven), it emits a UserWarning at construction
time.
+ import warnings
+
+ def unguarded_gt(x):
+ return x > 0
+
+ with self.sql_conf(_TRANSPILE_ON):
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ u = UserDefinedFunction(unguarded_gt, BooleanType())
+ null_guard_warnings = [w for w in caught if "NULL guard" in
str(w.message)]
+ self.assertTrue(u.transpiled, "unguarded_gt should transpile
(under strict or loose)")
+ self.assertTrue(null_guard_warnings, "Expected a UserWarning about
NULL guard")
+ self.assertIn("NULL guard", str(null_guard_warnings[0].message))
+
+ def test_udf_transpile_null_narrowing_len(self):
+ # A guarded len() call should skip the raise_error null guard.
+ def len_guarded(x):
+ if x is not None:
+ return len(x) > 3
+ return None
+
+ def len_early_return(x):
+ if x is None:
+ return None
+ return len(x) > 3
+
+ with self.sql_conf(_TRANSPILE_ON):
+ for name, func in [
+ ("len_guarded", len_guarded),
+ ("len_early_return", len_early_return),
+ ]:
+ with self.subTest(func=name):
+ u, warnings_text = self._udf_and_warnings(func,
BooleanType())
+ self.assertTrue(u.transpiled, f"{name} did not transpile:
{warnings_text}")
+ df = self.spark.createDataFrame([("hi",), ("hello",),
(None,)], "a string")
+ plan = self._optimized_plan(df.select(u("a")))
+ self.assertNotIn("raise_error", plan, f"{name}: plan had
raise_error")
+ self.assertEqual(
+ [False, True, None],
+ [r[0] for r in df.select(u("a")).collect()],
+ name,
+ )
+
if __name__ == "__main__":
from pyspark.testing import main
diff --git a/python/pyspark/sql/transpile.py b/python/pyspark/sql/transpile.py
index 9d371a3d64ca..bef13efe9a7d 100644
--- a/python/pyspark/sql/transpile.py
+++ b/python/pyspark/sql/transpile.py
@@ -203,6 +203,9 @@ def _is_definitely_basic_type(node: ast.AST) -> bool:
return _is_definitely_basic_type(operand)
case ast.Name():
return True
+ case ast.Call(func=ast.Name(id="len"), args=[_], keywords=[]):
+ # len() always returns an int: CPython raises if __len__ returns
anything else.
+ return True
case _:
return False
@@ -256,6 +259,46 @@ def _truthiness_col(cat: Optional[str], c: Column) ->
Optional[Column]:
return None
+def _null_narrowed(params: List[str], test_node: ast.AST) -> Tuple[frozenset,
frozenset]:
+ """Return ``(body_nonnull, else_nonnull)`` — frozensets of parameter names
+ that are proven non-null in the true branch and the false branch
+ respectively of an ``if``/ternary with this test.
+
+ Recognised patterns (``p`` must be in ``params``):
+ ``p is not None`` → body: {p}
+ ``p is None`` → else: {p}
+ ``None is not p`` → body: {p}
+ ``None is p`` → else: {p}
+ ``A and B and …`` → body: union of each clause's body set
+ All other nodes yield empty sets (conservative; no narrowing assumed).
+ """
+ match test_node:
+ case (
+ ast.Compare(
+ left=ast.Name(id=name),
+ ops=[op],
+ comparators=[ast.Constant(value=None)],
+ )
+ | ast.Compare(
+ left=ast.Constant(value=None),
+ ops=[op],
+ comparators=[ast.Name(id=name)],
+ )
+ ) if name in params:
+ if isinstance(op, ast.IsNot):
+ return frozenset({name}), frozenset()
+ if isinstance(op, ast.Is):
+ return frozenset(), frozenset({name})
+ case ast.BoolOp(op=ast.And(), values=values):
+ # x is not None and y is not None and … → body: union, else: {}
+ body_nonnull: frozenset = frozenset()
+ for v in values:
+ bn, _ = _null_narrowed(params, v)
+ body_nonnull |= bn
+ return body_nonnull, frozenset()
+ return frozenset(), frozenset()
+
+
class CatalystTranspiler(AbstractTranspiler):
"""Transpiler that attempts to convert a Python UDF into native Spark SQL
expressions."""
@@ -264,6 +307,8 @@ class CatalystTranspiler(AbstractTranspiler):
def __init__(self) -> None:
self._param_categories: dict[int, str] = {}
self._category_cache: dict[int, str] = {}
+ self._nonnull_params: frozenset = frozenset()
+ self._null_guard_emitted: bool = False
# TODO (SPARK-55218): handle implicit-None return bodies like
# ``def f(x): x + x`` -- no return statement means return None;
@@ -284,6 +329,24 @@ class CatalystTranspiler(AbstractTranspiler):
return lit(None)
return self._convert_chunk(params, statements[0])
+ def _is_nonnull_node(self, params: List[str], node: ast.AST) -> bool:
+ """Return True when ``node`` is statically known to produce a non-null
value.
+
+ A non-None literal constant is always non-null. A parameter name is
+ non-null when proven so by an enclosing ``is not None`` guard recorded
+ in ``self._nonnull_params``. ``len(x)`` is non-null when ``x`` is
non-null
+ because ``length`` of a non-null string always returns a non-null
integer.
+ """
+ match node:
+ case ast.Constant(value=v):
+ return v is not None
+ case ast.Name(id=name) if name in params:
+ return name in self._nonnull_params
+ case ast.Call(func=ast.Name(id="len"), args=[arg], keywords=[]):
+ return self._is_nonnull_node(params, arg)
+ case _:
+ return False
+
def _safe_category(self, params: List[str], node: Optional[ast.AST]) ->
Optional[str]:
"""Best-effort input-type category for an if/else branch, or ``None``
when
it can't be pinned down statically.
@@ -501,12 +564,15 @@ class CatalystTranspiler(AbstractTranspiler):
)
left_col = self._convert_chunk(params, left_node)
right_col = self._convert_chunk(params, right_node)
+ if self._is_nonnull_node(params, left_node) and
self._is_nonnull_node(params, right_node):
+ return op(left_col, right_col)
null_guard = left_col.isNull() | right_col.isNull()
err = lit(
"Python UDF transpiler: cannot compare NULL with operator "
f"`{op_repr}`; Python would raise TypeError here. Add an "
"`is not None` guard or filter NULLs upstream."
)
+ self._null_guard_emitted = True
return when(null_guard, raise_error(err)).otherwise(op(left_col,
right_col))
def _category(self, params: List[str], node: ast.AST) -> str:
@@ -732,21 +798,31 @@ class CatalystTranspiler(AbstractTranspiler):
case ast.IfExp(test=test, body=body_expr, orelse=orelse_expr):
# Ternary `body if test else orelse` -- shares the
# NULL-as-falsy lowering with the if-statement case.
+ test_col = self._convert_chunk(params, test)
+ body_nonnull, else_nonnull = _null_narrowed(params, test)
+ saved_nonnull = self._nonnull_params
+ self._nonnull_params = saved_nonnull | body_nonnull
+ body_col = self._convert_chunk(params, body_expr)
+ self._nonnull_params = saved_nonnull | else_nonnull
+ else_col = self._convert_chunk(params, orelse_expr)
+ self._nonnull_params = saved_nonnull
return self._convert_if_like(
- params,
- self._convert_chunk(params, test),
- self._convert_chunk(params, body_expr),
- self._convert_chunk(params, orelse_expr),
- test,
- body_expr,
- orelse_expr,
+ params, test_col, body_col, else_col, test, body_expr,
orelse_expr
)
case ast.If(test, success, orelse):
+ test_col = self._convert_chunk(params, test)
+ body_nonnull, else_nonnull = _null_narrowed(params, test)
+ saved_nonnull = self._nonnull_params
+ self._nonnull_params = saved_nonnull | body_nonnull
+ body_col = self._convert_branch(params, success, "body")
+ self._nonnull_params = saved_nonnull | else_nonnull
+ else_col = self._convert_branch(params, orelse, "else body")
+ self._nonnull_params = saved_nonnull
return self._convert_if_like(
params,
- self._convert_chunk(params, test),
- self._convert_branch(params, success, "body"),
- self._convert_branch(params, orelse, "else body"),
+ test_col,
+ body_col,
+ else_col,
test,
success[0] if success else None,
orelse[0] if orelse else None,
@@ -908,11 +984,14 @@ class CatalystTranspiler(AbstractTranspiler):
"types fall back to interpreted Python"
)
arg_col = self._convert_chunk(params, arg)
+ if self._is_nonnull_node(params, arg):
+ return length(arg_col)
err = lit(
"Python UDF transpiler: cannot call len() on NULL; "
"Python would raise TypeError here. Add an "
"`is not None` guard or filter NULLs upstream."
)
+ self._null_guard_emitted = True
return when(arg_col.isNull(),
raise_error(err)).otherwise(length(arg_col))
case _:
raise UnsupportedOperationException(
@@ -938,7 +1017,20 @@ class CatalystTranspiler(AbstractTranspiler):
# Category inference depends on the per-variant assumptions above.
Cache
# each AST node only for this lowering so recursive conversion stays
linear.
self._category_cache = {}
+ # Reset per-transpilation null-narrowing state.
+ self._nonnull_params = frozenset()
+ self._null_guard_emitted = False
function_body = function_ast.body
+ # Normalize `if <test>: return y\nreturn body` (2-stmt early-return)
+ # into a single if/else so null narrowing can apply to the main body.
+ match function_body:
+ case [
+ ast.If(test=test, body=[ast.Return() as early_ret], orelse=[])
as if_node,
+ ast.Return() as main_ret,
+ ]:
+ synthesized = ast.If(test=test, body=[early_ret],
orelse=[main_ret])
+ ast.fix_missing_locations(ast.copy_location(synthesized,
if_node))
+ function_body = [synthesized]
if len(function_body) != 1:
raise UnsupportedOperationException(
"functions with more than one top-level statement are not "
@@ -991,6 +1083,15 @@ class CatalystTranspiler(AbstractTranspiler):
"interpreted Python"
)
converted = self._convert_chunk(params, function_body[0])
+ if self._null_guard_emitted:
+ warnings.warn(
+ "Python UDF transpiler: the transpiled UDF includes a NULL
guard "
+ "that raises at runtime when a NULL input is encountered. Add
an "
+ "`if x is not None` guard around comparisons, or filter NULLs "
+ "upstream, to avoid this.",
+ UserWarning,
+ stacklevel=2,
+ )
# Cast to the declared return type so the rewritten plan reports a
# known data type to the optimizer's plan validator (otherwise it
# sees an UnresolvedFunction tree and reports VOID, which fails
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]