jzhan-2026 commented on code in PR #59108:
URL: https://github.com/apache/spark/pull/59108#discussion_r4383627916
##########
python/pyspark/sql/transpile.py:
##########
@@ -181,6 +181,8 @@ def _is_definitely_basic_type(node: ast.AST) -> bool:
return _is_definitely_basic_type(operand)
case ast.Name():
return True
+ case ast.Call(func=ast.Name(id="len"), args=[arg], keywords=[]):
+ return _is_definitely_basic_type(arg)
Review Comment:
Right, `len()` always returns an int (CPython raises `TypeError` if
`__len__` returns anything else), so I dropped the recursive check on the
argument and just return `True`.
##########
python/pyspark/sql/transpile.py:
##########
@@ -653,21 +718,31 @@ def _convert_chunk(self, params: List[str], body: ast.AST
| None) -> Column:
case ast.IfExp(test=test, body=body_expr, orelse=orelse_expr):
# Ternary `body if test else orelse` -- shares the
# NULL-as-falsy lowering with the if-statement case.
+ test_col = self._convert_chunk(params, test)
+ body_nonnull, else_nonnull = _null_narrowed(params, test)
+ saved = self._nonnull_params
Review Comment:
Done, renamed to `saved_nonnull` in both the ternary and if-statement
branches.
##########
python/pyspark/sql/transpile.py:
##########
@@ -912,6 +1003,15 @@ def _transpile_from_ast(
"interpreted Python"
)
converted = self._convert_chunk(params, function_body[0])
+ if self._null_guard_emitted:
+ warnings.warn(
+ "Python UDF transpiler: the transpiled UDF includes a NULL
guard "
+ "that raises at runtime when a NULL input is encountered. Add
an "
+ "`if x is not None` guard around comparisons, or filter NULLs "
+ "upstream, to avoid this.",
+ UserWarning,
+ stacklevel=2,
+ )
Review Comment:
Thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
