This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 44efa48dd95e [SPARK-54623][PYTHON][TEST] Add coverage test for UDTF
null checker
44efa48dd95e is described below
commit 44efa48dd95e7c6a3f3c664566ec1dd4154b7851
Author: Tian Gao <[email protected]>
AuthorDate: Mon Dec 8 07:43:06 2025 +0900
[SPARK-54623][PYTHON][TEST] Add coverage test for UDTF null checker
### What changes were proposed in this pull request?
Coverage tests are added for the null checker in UDTF.
### Why are the changes needed?
Improve test coverage and I need to refactor the code later so we need to
make sure there's no regression.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
The test passed locally.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53362 from gaogaotiantian/test-null-checker.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/sql/tests/test_udtf.py | 105 ++++++++++++++++++++++++++++++++++
1 file changed, 105 insertions(+)
diff --git a/python/pyspark/sql/tests/test_udtf.py
b/python/pyspark/sql/tests/test_udtf.py
index 72bef492e23c..eeb07400b060 100644
--- a/python/pyspark/sql/tests/test_udtf.py
+++ b/python/pyspark/sql/tests/test_udtf.py
@@ -812,6 +812,111 @@ class BaseUDTFTestsMixin:
with self.assertRaisesRegex(err_type, expected):
func().collect()
+ def test_udtf_nullable_check(self):
+ for ret_type, value, expected in [
+ (
+ StructType([StructField("value", ArrayType(IntegerType(),
False))]),
+ ([None],),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType([StructField("value", ArrayType(IntegerType(),
True))]),
+ ([None],),
+ [Row(value=[None])],
+ ),
+ (
+ StructType([StructField("value", MapType(StringType(),
IntegerType(), False))]),
+ ({"a": None},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType([StructField("value", MapType(StringType(),
IntegerType(), True))]),
+ ({"a": None},),
+ [Row(value={"a": None})],
+ ),
+ (
+ StructType([StructField("value", MapType(StringType(),
IntegerType(), True))]),
+ ({None: 1},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType([StructField("value", MapType(StringType(),
IntegerType(), False))]),
+ ({None: 1},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType(
+ [
+ StructField(
+ "value", MapType(StringType(),
ArrayType(IntegerType(), False), False)
+ )
+ ]
+ ),
+ ({"s": [None]},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType(
+ [
+ StructField(
+ "value",
+ MapType(
+ StructType([StructField("value", StringType(),
False)]),
+ IntegerType(),
+ False,
+ ),
+ )
+ ]
+ ),
+ ({(None,): 1},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType(
+ [
+ StructField(
+ "value",
+ MapType(
+ StructType([StructField("value", StringType(),
False)]),
+ IntegerType(),
+ True,
+ ),
+ )
+ ]
+ ),
+ ({(None,): 1},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType(
+ [StructField("value", StructType([StructField("value",
StringType(), False)]))]
+ ),
+ ((None,),),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType(
+ [
+ StructField(
+ "value",
+ StructType(
+ [StructField("value", ArrayType(StringType(),
False), False)]
+ ),
+ )
+ ]
+ ),
+ (([None],),),
+ "PySparkRuntimeError",
+ ),
+ ]:
+
+ class TestUDTF:
+ def eval(self):
+ yield value
+
+ with self.subTest(ret_type=ret_type, value=value):
+ self._check_result_or_exception(TestUDTF, ret_type, expected)
+
def test_numeric_output_type_casting(self):
class TestUDTF:
def eval(self):
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
