MaxGekk commented on code in PR #48455:
URL: https://github.com/apache/spark/pull/48455#discussion_r1804834686
##########
python/pyspark/sql/functions/builtin.py:
##########
@@ -11197,6 +11197,127 @@ def encode(col: "ColumnOrName", charset: str) ->
Column:
return _invoke_function("encode", _to_java_column(col),
_enum_to_value(charset))
+@_try_remote_functions
+def is_valid_utf8(str: "ColumnOrName") -> Column:
+ """
+ Returns true if the input is a valid UTF-8 string, otherwise returns false.
+
+ .. versionadded:: 4.0.0
+
+ Parameters
+ ----------
+ str : :class:`~pyspark.sql.Column` or str
+ A column of strings, each representing a URL-encoded string.
+
+ Returns
+ -------
+ :class:`~pyspark.sql.Column`
+ whether the input string is a valid UTF-8 string.
+
+ Examples
+ --------
+ >>> import pyspark.sql.functions as sf
+ >>> spark.range(1).select(sf.is_valid_utf8(sf.lit("SparkSQL"))).show()
+ +-----------------------+
+ |is_valid_utf8(SparkSQL)|
+ +-----------------------+
+ | true|
+ +-----------------------+
+ """
+ return _invoke_function_over_columns("is_valid_utf8", str)
+
+
+@_try_remote_functions
+def make_valid_utf8(str: "ColumnOrName") -> Column:
+ """
+ Returns a new string in which all invalid UTF-8 byte sequences, if any,
are replaced by the
+ Unicode replacement character (U+FFFD).
+
+ .. versionadded:: 4.0.0
+
+ Parameters
+ ----------
+ str : :class:`~pyspark.sql.Column` or str
+ A column of strings, each representing a URL-encoded string.
+
+ Returns
+ -------
+ :class:`~pyspark.sql.Column`
+ the valid UTF-8 version of the given input string.
+
+ Examples
+ --------
+ >>> import pyspark.sql.functions as sf
+ >>> spark.range(1).select(sf.make_valid_utf8(sf.lit("SparkSQL"))).show()
+ +-------------------------+
+ |make_valid_utf8(SparkSQL)|
+ +-------------------------+
+ | SparkSQL|
+ +-------------------------+
+ """
+ return _invoke_function_over_columns("make_valid_utf8", str)
+
+
+@_try_remote_functions
+def validate_utf8(str: "ColumnOrName") -> Column:
+ """
+ Returns the input value if it corresponds to a valid UTF-8 string, or
emits an error otherwise.
+
+ .. versionadded:: 4.0.0
+
+ Parameters
+ ----------
+ str : :class:`~pyspark.sql.Column` or str
+ A column of strings, each representing a URL-encoded string.
Review Comment:
URL-encoded string? really?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
