This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new dc73d212ae62 [SPARK-58628][SQL] Transpiler: skip null guards for 
is-not-None-narrowed parameters
dc73d212ae62 is described below

commit dc73d212ae6213f0079ffd01dce7dad99e7db827
Author: jzhan-2026 <[email protected]>
AuthorDate: Fri Oct 30 17:05:38 2026 -0700

    [SPARK-58628][SQL] Transpiler: skip null guards for is-not-None-narrowed 
parameters
    
    Three related improvements to the Python UDF transpiler:
    
    1. **Null narrowing.** When a parameter is guarded by `x is not None` (or 
`x is None`) in an `if`/ternary test, the transpiler tracks that the parameter 
is proven non-null in the appropriate branch. Ordering comparisons (`<`, `<=`, 
`>`, `>=`) and `len()` calls on such parameters skip the `raise_error` null 
guard entirely.
    
    2. **2-statement early-return normalization.** A common guard idiom:
       ```python
       if x is None:
           return default
       return x > 0
    
       is normalized into a single if/else before lowering, so null narrowing 
applies to the main body.
    
    3. UserWarning on null guard emission. When a null guard is still emitted 
(non-null cannot be proven), the transpiler warns at UDF construction time.
    
    Co-authored with jing.zhansnowflake.com
    
    Any ordering comparison in a transpiled UDF previously always emitted a 
raise_error guard, even when the code already checked x is not None. This 
produced unnecessary plan nodes and blocked clean transpilation of the most 
common Python null-guard idiom.
    
    Yes. UDFs with is not None-guarded comparisons produce leaner plans (no 
raise_error node in the guarded branch). The UserWarning at construction time 
is new when unguarded comparisons remain. Behavior on non-null inputs is 
unchanged.
    
    Four new test cases in test_udf_transpile_unit.py:
    - test_udf_transpile_null_narrowing_skips_null_guard — if/ternary with is 
not None guard: plan has no raise_error, results match Python
    - test_udf_transpile_early_return_null_guard — 2-stmt early-return pattern 
transpiles with no raise_error
    - test_udf_transpile_null_guard_warning — unguarded comparison emits 
UserWarning
    - test_udf_transpile_null_narrowing_len — guarded len() has no raise_error 
in plan
    
    Co-authored by: Claude Sonnet 4.6
    
    Closes #59108 from jzhan-2026/spark-58628-non-nullable-null-guard.
    
    Authored-by: jzhan-2026 <[email protected]>
    Co-authored-by: [email protected] <[email protected]>
    Signed-off-by: Holden Karau <[email protected]>
---
 .../pyspark/sql/tests/test_udf_transpile_unit.py   | 144 ++++++++++++++++++++-
 python/pyspark/sql/transpile.py                    | 121 +++++++++++++++--
 2 files changed, 254 insertions(+), 11 deletions(-)

diff --git a/python/pyspark/sql/tests/test_udf_transpile_unit.py 
b/python/pyspark/sql/tests/test_udf_transpile_unit.py
index 5c0309bb53b3..c19c38814fff 100644
--- a/python/pyspark/sql/tests/test_udf_transpile_unit.py
+++ b/python/pyspark/sql/tests/test_udf_transpile_unit.py
@@ -246,7 +246,7 @@ class UDFTranspileUnitTests(ReusedSQLTestCase):
             if x is not None:
                 return x << 1
 
-        def multi_statement(x):  # > 1 top-level statement, not handled.
+        def multi_statement(x):  # First stmt is an assignment, not an 
if-guard.
             y = 1
             return x + y if x is not None else 0
 
@@ -2750,6 +2750,148 @@ class UDFTranspileUnitTests(ReusedSQLTestCase):
         self.assertEqual(len(combos_u), 3)
         self.assertEqual({c[0] for c in combos_u}, {"integer", "float", 
"string"})
 
+    def test_udf_transpile_null_narrowing_skips_null_guard(self):
+        # When a parameter is guarded by `x is not None` in an
+        # if/ternary test, the transpiler should skip the raise_error null 
guard
+        # for ordering comparisons on that parameter within the guarded branch.
+        def if_guard_gt(x):
+            return (x > 0) if x is not None else None
+
+        def if_guard_and_lt(a, b):
+            return (a < b) if a is not None and b is not None else None
+
+        def ternary_guard_ge(x):
+            return (x >= 0) if x is not None else None
+
+        def is_none_ternary(x):
+            return None if x is None else x > 0
+
+        cases_if = [
+            # if/else with `is not None` guard
+            ("if_guard_gt", if_guard_gt),
+            # if/else with `is not None` and-chain (two params)
+            ("if_guard_and_lt", if_guard_and_lt),
+            # ternary with `is not None` guard
+            ("ternary_guard_ge", ternary_guard_ge),
+            # `is None` guard: non-null in the else branch
+            ("is_none_ternary", is_none_ternary),
+        ]
+        with self.sql_conf(_TRANSPILE_ON):
+            for name, func in cases_if:
+                with self.subTest(func=name):
+                    u, warnings_text = self._udf_and_warnings(func, 
BooleanType())
+                    self.assertTrue(u.transpiled, f"{name} did not transpile: 
{warnings_text}")
+                    arity = func.__code__.co_argcount
+                    if arity == 1:
+                        df = self.spark.createDataFrame([(1,), (-1,), 
(None,)], "a long")
+                        plan = self._optimized_plan(df.select(u("a")))
+                        self.assertNotIn("raise_error", plan, f"{name}: plan 
had raise_error")
+                        self.assertEqual(
+                            [True, False, None],
+                            [r[0] for r in df.select(u("a")).collect()],
+                            name,
+                        )
+                    else:
+                        df = self.spark.createDataFrame(
+                            [(1, 2), (2, 1), (None, 1)], "a long, b long"
+                        )
+                        plan = self._optimized_plan(df.select(u("a", "b")))
+                        self.assertNotIn("raise_error", plan, f"{name}: plan 
had raise_error")
+                        self.assertEqual(
+                            [True, False, None],
+                            [r[0] for r in df.select(u("a", "b")).collect()],
+                            name,
+                        )
+
+    def test_udf_transpile_early_return_null_guard(self):
+        # A 2-statement early-return pattern
+        # `if x is None: return y\nreturn body` is normalized to a single
+        # if/else so null narrowing applies to the main body.
+        def early_return_gt(x):
+            if x is None:
+                return None
+            return x > 0
+
+        def early_return_lt_default(x):
+            if x is None:
+                return False
+            return x < 10
+
+        with self.sql_conf(_TRANSPILE_ON):
+            for name, func, rt, rows, expected in [
+                (
+                    "early_return_gt",
+                    early_return_gt,
+                    BooleanType(),
+                    [(1,), (-1,), (None,)],
+                    [True, False, None],
+                ),
+                (
+                    "early_return_lt_default",
+                    early_return_lt_default,
+                    BooleanType(),
+                    [(5,), (15,), (None,)],
+                    [True, False, False],
+                ),
+            ]:
+                with self.subTest(func=name):
+                    u, warnings_text = self._udf_and_warnings(func, rt)
+                    self.assertTrue(u.transpiled, f"{name} did not transpile: 
{warnings_text}")
+                    df = self.spark.createDataFrame(rows, "a long")
+                    plan = self._optimized_plan(df.select(u("a")))
+                    self.assertNotIn("raise_error", plan, f"{name}: plan had 
raise_error")
+                    self.assertEqual(
+                        expected,
+                        [r[0] for r in df.select(u("a")).collect()],
+                        name,
+                    )
+
+    def test_udf_transpile_null_guard_warning(self):
+        # When the transpiler still emits a null guard (because
+        # non-null cannot be proven), it emits a UserWarning at construction 
time.
+        import warnings
+
+        def unguarded_gt(x):
+            return x > 0
+
+        with self.sql_conf(_TRANSPILE_ON):
+            with warnings.catch_warnings(record=True) as caught:
+                warnings.simplefilter("always")
+                u = UserDefinedFunction(unguarded_gt, BooleanType())
+            null_guard_warnings = [w for w in caught if "NULL guard" in 
str(w.message)]
+            self.assertTrue(u.transpiled, "unguarded_gt should transpile 
(under strict or loose)")
+            self.assertTrue(null_guard_warnings, "Expected a UserWarning about 
NULL guard")
+            self.assertIn("NULL guard", str(null_guard_warnings[0].message))
+
+    def test_udf_transpile_null_narrowing_len(self):
+        # A guarded len() call should skip the raise_error null guard.
+        def len_guarded(x):
+            if x is not None:
+                return len(x) > 3
+            return None
+
+        def len_early_return(x):
+            if x is None:
+                return None
+            return len(x) > 3
+
+        with self.sql_conf(_TRANSPILE_ON):
+            for name, func in [
+                ("len_guarded", len_guarded),
+                ("len_early_return", len_early_return),
+            ]:
+                with self.subTest(func=name):
+                    u, warnings_text = self._udf_and_warnings(func, 
BooleanType())
+                    self.assertTrue(u.transpiled, f"{name} did not transpile: 
{warnings_text}")
+                    df = self.spark.createDataFrame([("hi",), ("hello",), 
(None,)], "a string")
+                    plan = self._optimized_plan(df.select(u("a")))
+                    self.assertNotIn("raise_error", plan, f"{name}: plan had 
raise_error")
+                    self.assertEqual(
+                        [False, True, None],
+                        [r[0] for r in df.select(u("a")).collect()],
+                        name,
+                    )
+
 
 if __name__ == "__main__":
     from pyspark.testing import main
diff --git a/python/pyspark/sql/transpile.py b/python/pyspark/sql/transpile.py
index 9d371a3d64ca..bef13efe9a7d 100644
--- a/python/pyspark/sql/transpile.py
+++ b/python/pyspark/sql/transpile.py
@@ -203,6 +203,9 @@ def _is_definitely_basic_type(node: ast.AST) -> bool:
             return _is_definitely_basic_type(operand)
         case ast.Name():
             return True
+        case ast.Call(func=ast.Name(id="len"), args=[_], keywords=[]):
+            # len() always returns an int: CPython raises if __len__ returns 
anything else.
+            return True
         case _:
             return False
 
@@ -256,6 +259,46 @@ def _truthiness_col(cat: Optional[str], c: Column) -> 
Optional[Column]:
     return None
 
 
+def _null_narrowed(params: List[str], test_node: ast.AST) -> Tuple[frozenset, 
frozenset]:
+    """Return ``(body_nonnull, else_nonnull)`` — frozensets of parameter names
+    that are proven non-null in the true branch and the false branch
+    respectively of an ``if``/ternary with this test.
+
+    Recognised patterns (``p`` must be in ``params``):
+      ``p is not None``          → body: {p}
+      ``p is None``              → else: {p}
+      ``None is not p``          → body: {p}
+      ``None is p``              → else: {p}
+      ``A and B and …``          → body: union of each clause's body set
+    All other nodes yield empty sets (conservative; no narrowing assumed).
+    """
+    match test_node:
+        case (
+            ast.Compare(
+                left=ast.Name(id=name),
+                ops=[op],
+                comparators=[ast.Constant(value=None)],
+            )
+            | ast.Compare(
+                left=ast.Constant(value=None),
+                ops=[op],
+                comparators=[ast.Name(id=name)],
+            )
+        ) if name in params:
+            if isinstance(op, ast.IsNot):
+                return frozenset({name}), frozenset()
+            if isinstance(op, ast.Is):
+                return frozenset(), frozenset({name})
+        case ast.BoolOp(op=ast.And(), values=values):
+            # x is not None and y is not None and … → body: union, else: {}
+            body_nonnull: frozenset = frozenset()
+            for v in values:
+                bn, _ = _null_narrowed(params, v)
+                body_nonnull |= bn
+            return body_nonnull, frozenset()
+    return frozenset(), frozenset()
+
+
 class CatalystTranspiler(AbstractTranspiler):
     """Transpiler that attempts to convert a Python UDF into native Spark SQL 
expressions."""
 
@@ -264,6 +307,8 @@ class CatalystTranspiler(AbstractTranspiler):
     def __init__(self) -> None:
         self._param_categories: dict[int, str] = {}
         self._category_cache: dict[int, str] = {}
+        self._nonnull_params: frozenset = frozenset()
+        self._null_guard_emitted: bool = False
 
     # TODO (SPARK-55218): handle implicit-None return bodies like
     # ``def f(x): x + x`` -- no return statement means return None;
@@ -284,6 +329,24 @@ class CatalystTranspiler(AbstractTranspiler):
             return lit(None)
         return self._convert_chunk(params, statements[0])
 
+    def _is_nonnull_node(self, params: List[str], node: ast.AST) -> bool:
+        """Return True when ``node`` is statically known to produce a non-null 
value.
+
+        A non-None literal constant is always non-null. A parameter name is
+        non-null when proven so by an enclosing ``is not None`` guard recorded
+        in ``self._nonnull_params``. ``len(x)`` is non-null when ``x`` is 
non-null
+        because ``length`` of a non-null string always returns a non-null 
integer.
+        """
+        match node:
+            case ast.Constant(value=v):
+                return v is not None
+            case ast.Name(id=name) if name in params:
+                return name in self._nonnull_params
+            case ast.Call(func=ast.Name(id="len"), args=[arg], keywords=[]):
+                return self._is_nonnull_node(params, arg)
+            case _:
+                return False
+
     def _safe_category(self, params: List[str], node: Optional[ast.AST]) -> 
Optional[str]:
         """Best-effort input-type category for an if/else branch, or ``None`` 
when
         it can't be pinned down statically.
@@ -501,12 +564,15 @@ class CatalystTranspiler(AbstractTranspiler):
             )
         left_col = self._convert_chunk(params, left_node)
         right_col = self._convert_chunk(params, right_node)
+        if self._is_nonnull_node(params, left_node) and 
self._is_nonnull_node(params, right_node):
+            return op(left_col, right_col)
         null_guard = left_col.isNull() | right_col.isNull()
         err = lit(
             "Python UDF transpiler: cannot compare NULL with operator "
             f"`{op_repr}`; Python would raise TypeError here. Add an "
             "`is not None` guard or filter NULLs upstream."
         )
+        self._null_guard_emitted = True
         return when(null_guard, raise_error(err)).otherwise(op(left_col, 
right_col))
 
     def _category(self, params: List[str], node: ast.AST) -> str:
@@ -732,21 +798,31 @@ class CatalystTranspiler(AbstractTranspiler):
             case ast.IfExp(test=test, body=body_expr, orelse=orelse_expr):
                 # Ternary `body if test else orelse` -- shares the
                 # NULL-as-falsy lowering with the if-statement case.
+                test_col = self._convert_chunk(params, test)
+                body_nonnull, else_nonnull = _null_narrowed(params, test)
+                saved_nonnull = self._nonnull_params
+                self._nonnull_params = saved_nonnull | body_nonnull
+                body_col = self._convert_chunk(params, body_expr)
+                self._nonnull_params = saved_nonnull | else_nonnull
+                else_col = self._convert_chunk(params, orelse_expr)
+                self._nonnull_params = saved_nonnull
                 return self._convert_if_like(
-                    params,
-                    self._convert_chunk(params, test),
-                    self._convert_chunk(params, body_expr),
-                    self._convert_chunk(params, orelse_expr),
-                    test,
-                    body_expr,
-                    orelse_expr,
+                    params, test_col, body_col, else_col, test, body_expr, 
orelse_expr
                 )
             case ast.If(test, success, orelse):
+                test_col = self._convert_chunk(params, test)
+                body_nonnull, else_nonnull = _null_narrowed(params, test)
+                saved_nonnull = self._nonnull_params
+                self._nonnull_params = saved_nonnull | body_nonnull
+                body_col = self._convert_branch(params, success, "body")
+                self._nonnull_params = saved_nonnull | else_nonnull
+                else_col = self._convert_branch(params, orelse, "else body")
+                self._nonnull_params = saved_nonnull
                 return self._convert_if_like(
                     params,
-                    self._convert_chunk(params, test),
-                    self._convert_branch(params, success, "body"),
-                    self._convert_branch(params, orelse, "else body"),
+                    test_col,
+                    body_col,
+                    else_col,
                     test,
                     success[0] if success else None,
                     orelse[0] if orelse else None,
@@ -908,11 +984,14 @@ class CatalystTranspiler(AbstractTranspiler):
                         "types fall back to interpreted Python"
                     )
                 arg_col = self._convert_chunk(params, arg)
+                if self._is_nonnull_node(params, arg):
+                    return length(arg_col)
                 err = lit(
                     "Python UDF transpiler: cannot call len() on NULL; "
                     "Python would raise TypeError here. Add an "
                     "`is not None` guard or filter NULLs upstream."
                 )
+                self._null_guard_emitted = True
                 return when(arg_col.isNull(), 
raise_error(err)).otherwise(length(arg_col))
             case _:
                 raise UnsupportedOperationException(
@@ -938,7 +1017,20 @@ class CatalystTranspiler(AbstractTranspiler):
         # Category inference depends on the per-variant assumptions above. 
Cache
         # each AST node only for this lowering so recursive conversion stays 
linear.
         self._category_cache = {}
+        # Reset per-transpilation null-narrowing state.
+        self._nonnull_params = frozenset()
+        self._null_guard_emitted = False
         function_body = function_ast.body
+        # Normalize `if <test>: return y\nreturn body` (2-stmt early-return)
+        # into a single if/else so null narrowing can apply to the main body.
+        match function_body:
+            case [
+                ast.If(test=test, body=[ast.Return() as early_ret], orelse=[]) 
as if_node,
+                ast.Return() as main_ret,
+            ]:
+                synthesized = ast.If(test=test, body=[early_ret], 
orelse=[main_ret])
+                ast.fix_missing_locations(ast.copy_location(synthesized, 
if_node))
+                function_body = [synthesized]
         if len(function_body) != 1:
             raise UnsupportedOperationException(
                 "functions with more than one top-level statement are not "
@@ -991,6 +1083,15 @@ class CatalystTranspiler(AbstractTranspiler):
                     "interpreted Python"
                 )
         converted = self._convert_chunk(params, function_body[0])
+        if self._null_guard_emitted:
+            warnings.warn(
+                "Python UDF transpiler: the transpiled UDF includes a NULL 
guard "
+                "that raises at runtime when a NULL input is encountered. Add 
an "
+                "`if x is not None` guard around comparisons, or filter NULLs "
+                "upstream, to avoid this.",
+                UserWarning,
+                stacklevel=2,
+            )
         # Cast to the declared return type so the rewritten plan reports a
         # known data type to the optimizer's plan validator (otherwise it
         # sees an UnresolvedFunction tree and reports VOID, which fails


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to