zhengruifeng commented on code in PR #41534:
URL: https://github.com/apache/spark/pull/41534#discussion_r1225026809
##########
python/pyspark/sql/functions.py:
##########
@@ -11157,6 +11157,136 @@ def udf(
return _create_py_udf(f=f, returnType=returnType, useArrow=useArrow)
+# --------------- Conditional functions ------------------------
+
+
+@try_remote_functions
+def ifnull(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
+ """
+ Returns `col2` if `col1` is null, or `col1` otherwise.
+
+ .. versionadded:: 3.5.0
+
+ Parameters
+ ----------
+ col1 : :class:`~pyspark.sql.Column` or str
+ col2 : :class:`~pyspark.sql.Column` or str
+
+ Examples
+ --------
+ >>> df = spark.createDataFrame([(None,), (1,)], ["e"])
+ >>> df.select(ifnull(df.e, lit(8)).alias('r')).collect()
+ [Row(r=8), Row(r=1)]
+ """
+ return _invoke_function_over_columns("ifnull", col1, col2)
+
+
+@try_remote_functions
+def isnotnull(col1: "ColumnOrName") -> Column:
Review Comment:
```suggestion
def isnotnull(col: "ColumnOrName") -> Column:
```
##########
connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala:
##########
@@ -5883,4 +5883,65 @@ object functions {
}
// scalastyle:off line.size.limit
+
//////////////////////////////////////////////////////////////////////////////////////////////
+ // Conditional Functions
Review Comment:
maybe name it after `misc non-aggregate functions`
https://github.com/apache/spark/blob/11390c509729f4fc1bfc9b23529a5d55a554b07a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala#L358
##########
python/pyspark/sql/functions.py:
##########
@@ -11157,6 +11157,142 @@ def udf(
return _create_py_udf(f=f, returnType=returnType, useArrow=useArrow)
+# --------------- Conditional functions ------------------------
+
Review Comment:
I think we should apply a different name in this case, like `map` in Scala
<-> `create_map` in Python.
cc @HyukjinKwon @beliefer do you have a good name for it?
##########
python/pyspark/sql/functions.py:
##########
@@ -11157,6 +11157,136 @@ def udf(
return _create_py_udf(f=f, returnType=returnType, useArrow=useArrow)
+# --------------- Conditional functions ------------------------
+
+
+@try_remote_functions
+def ifnull(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
+ """
+ Returns `col2` if `col1` is null, or `col1` otherwise.
+
+ .. versionadded:: 3.5.0
+
+ Parameters
+ ----------
+ col1 : :class:`~pyspark.sql.Column` or str
+ col2 : :class:`~pyspark.sql.Column` or str
+
+ Examples
+ --------
+ >>> df = spark.createDataFrame([(None,), (1,)], ["e"])
+ >>> df.select(ifnull(df.e, lit(8)).alias('r')).collect()
+ [Row(r=8), Row(r=1)]
+ """
+ return _invoke_function_over_columns("ifnull", col1, col2)
+
+
+@try_remote_functions
+def isnotnull(col1: "ColumnOrName") -> Column:
+ """
+ Returns true if `col` is not null, or false otherwise.
+
+ .. versionadded:: 3.5.0
+
+ Parameters
+ ----------
+ col1 : :class:`~pyspark.sql.Column` or str
+
+ Examples
+ --------
+ >>> df = spark.createDataFrame([(None,), (1,)], ["e"])
+ >>> df.select(isnotnull(df.e).alias('r')).collect()
+ [Row(r=False), Row(r=True)]
+ """
+ return _invoke_function_over_columns("isnotnull", col1)
+
+
+@try_remote_functions
+def equal_null(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
+ """
+ Returns same result as the EQUAL(=) operator for non-null operands,
+ but returns true if both are null, false if one of the them is null.
+
+ .. versionadded:: 3.5.0
+
+ Parameters
+ ----------
+ col1 : :class:`~pyspark.sql.Column` or str
+ col2 : :class:`~pyspark.sql.Column` or str
+
+ Examples
+ --------
+ >>> df = spark.createDataFrame([(None, None,), (1, 9,)], ["a", "b"])
+ >>> df.select(equal_null(df.a, df.b).alias('r')).collect()
+ [Row(r=True), Row(r=False)]
+ """
+ return _invoke_function_over_columns("equal_null", col1, col2)
+
+
+@try_remote_functions
+def nullif(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
+ """
+ Returns null if `expr1` equals to `expr2`, or `expr1` otherwise.
Review Comment:
```suggestion
Returns null if `col1` equals to `col2`, or `col1` otherwise.
```
##########
python/pyspark/sql/functions.py:
##########
@@ -11157,6 +11157,136 @@ def udf(
return _create_py_udf(f=f, returnType=returnType, useArrow=useArrow)
+# --------------- Conditional functions ------------------------
+
+
+@try_remote_functions
+def ifnull(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
Review Comment:
current `ifnull` use the same expression as `nvl`, let's discuss in
https://github.com/apache/spark/pull/41516#discussion_r1225030166 first
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
