This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new eaccadb0fa9 [SPARK-40142][PYTHON][SQL][FOLLOW-UP] Make pyspark.sql.functions examples self-contained (part 6, ~50 functions) eaccadb0fa9 is described below commit eaccadb0fa90ee5c5fa85b8495e62cdadadc8081 Author: Khalid Mammadov <khalidmammad...@gmail.com> AuthorDate: Thu Sep 8 18:11:49 2022 -0500 [SPARK-40142][PYTHON][SQL][FOLLOW-UP] Make pyspark.sql.functions examples self-contained (part 6, ~50 functions) ### What changes were proposed in this pull request? It's part of the Pyspark docstrings improvement series (https://github.com/apache/spark/pull/37592, https://github.com/apache/spark/pull/37662, https://github.com/apache/spark/pull/37686, https://github.com/apache/spark/pull/37786) In this PR I mainly covered missing parts in the docstrings adding some more examples where it needed. ### Why are the changes needed? To improve PySpark documentation ### Does this PR introduce _any_ user-facing change? Yes, documentation ### How was this patch tested? ``` PYTHON_EXECUTABLE=python3.9 ./dev/lint-python ./python/run-tests --testnames pyspark.sql.functions bundle exec jekyll build ``` Closes #37797 from khalidmammadov/docstrings_funcs_part_6. Authored-by: Khalid Mammadov <khalidmammad...@gmail.com> Signed-off-by: Sean Owen <sro...@gmail.com> --- python/pyspark/sql/functions.py | 608 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 597 insertions(+), 11 deletions(-) diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index 9e1a0e84be3..4df5b8140f9 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -4778,6 +4778,11 @@ def window( start 15 minutes past the hour, e.g. 12:15-13:15, 13:15-14:15... provide `startTime` as `15 minutes`. + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. + Examples -------- >>> import datetime @@ -4840,6 +4845,11 @@ def session_window(timeColumn: "ColumnOrName", gapDuration: Union[Column, str]) static value, e.g. `10 minutes`, `1 second`, or an expression/UDF that specifies gap duration dynamically based on the input row. + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. + Examples -------- >>> df = spark.createDataFrame([("2016-03-11 09:00:07", 1)]).toDF("date", "val") @@ -4871,6 +4881,16 @@ def crc32(col: "ColumnOrName") -> Column: Calculates the cyclic redundancy check value (CRC32) of a binary column and returns the value as a bigint. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to compute on. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. + .. versionadded:: 1.5.0 Examples @@ -4886,6 +4906,16 @@ def md5(col: "ColumnOrName") -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to compute on. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. + Examples -------- >>> spark.createDataFrame([('ABC',)], ['a']).select(md5('a').alias('hash')).collect() @@ -4899,6 +4929,16 @@ def sha1(col: "ColumnOrName") -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to compute on. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. + Examples -------- >>> spark.createDataFrame([('ABC',)], ['a']).select(sha1('a').alias('hash')).collect() @@ -4914,6 +4954,19 @@ def sha2(col: "ColumnOrName", numBits: int) -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to compute on. + numBits : int + the desired bit length of the result, which must have a + value of 224, 256, 384, 512, or 0 (which is equivalent to 256). + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. + Examples -------- >>> digests = df.select(sha2(df.name, 256).alias('s')).collect() @@ -4930,10 +4983,37 @@ def hash(*cols: "ColumnOrName") -> Column: .. versionadded:: 2.0.0 + Parameters + ---------- + cols : :class:`~pyspark.sql.Column` or str + one or more columns to compute on. + + Returns + ------- + :class:`~pyspark.sql.Column` + hash value as int column. + Examples -------- - >>> spark.createDataFrame([('ABC',)], ['a']).select(hash('a').alias('hash')).collect() - [Row(hash=-757602832)] + >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2']) + + Hash for one column + + >>> df.select(hash('c1').alias('hash')).show() + +----------+ + | hash| + +----------+ + |-757602832| + +----------+ + + Two or more columns + + >>> df.select(hash('c1', 'c2').alias('hash')).show() + +---------+ + | hash| + +---------+ + |599895104| + +---------+ """ return _invoke_function_over_seq_of_columns("hash", cols) @@ -4944,18 +5024,45 @@ def xxhash64(*cols: "ColumnOrName") -> Column: .. versionadded:: 3.0.0 + Parameters + ---------- + cols : :class:`~pyspark.sql.Column` or str + one or more columns to compute on. + + Returns + ------- + :class:`~pyspark.sql.Column` + hash value as long column. + Examples -------- - >>> spark.createDataFrame([('ABC',)], ['a']).select(xxhash64('a').alias('hash')).collect() - [Row(hash=4105715581806190027)] + >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2']) + + Hash for one column + + >>> df.select(xxhash64('c1').alias('hash')).show() + +-------------------+ + | hash| + +-------------------+ + |4105715581806190027| + +-------------------+ + + Two or more columns + + >>> df.select(xxhash64('c1', 'c2').alias('hash')).show() + +-------------------+ + | hash| + +-------------------+ + |3233247871021311208| + +-------------------+ """ return _invoke_function_over_seq_of_columns("xxhash64", cols) def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None) -> Column: """ - Returns null if the input column is true; throws an exception with the provided error message - otherwise. + Returns `null` if the input column is `true`; throws an exception + with the provided error message otherwise. .. versionadded:: 3.1.0 @@ -4963,20 +5070,27 @@ def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None ---------- col : :class:`~pyspark.sql.Column` or str column name or column that represents the input column to test - errMsg : :class:`~pyspark.sql.Column` or str + errMsg : :class:`~pyspark.sql.Column` or str, optional A Python string literal or column containing the error message + Returns + ------- + :class:`~pyspark.sql.Column` + `null` if the input column is `true` otherwise throws an error with specified message. + Examples -------- >>> df = spark.createDataFrame([(0,1)], ['a', 'b']) >>> df.select(assert_true(df.a < df.b).alias('r')).collect() [Row(r=None)] - >>> df = spark.createDataFrame([(0,1)], ['a', 'b']) >>> df.select(assert_true(df.a < df.b, df.a).alias('r')).collect() [Row(r=None)] - >>> df = spark.createDataFrame([(0,1)], ['a', 'b']) >>> df.select(assert_true(df.a < df.b, 'error').alias('r')).collect() [Row(r=None)] + >>> df.select(assert_true(df.a > df.b, 'My error msg').alias('r')).collect() # doctest: +SKIP + ... + java.lang.RuntimeException: My error msg + ... """ if errMsg is None: return _invoke_function_over_columns("assert_true", col) @@ -4999,6 +5113,19 @@ def raise_error(errMsg: Union[Column, str]) -> Column: ---------- errMsg : :class:`~pyspark.sql.Column` or str A Python string literal or column containing the error message + + Returns + ------- + :class:`~pyspark.sql.Column` + throws an error with specified message. + + Examples + -------- + >>> df = spark.range(1) + >>> df.select(raise_error("My error message")).show() # doctest: +SKIP + ... + java.lang.RuntimeException: My error message + ... """ if not isinstance(errMsg, (str, Column)): raise TypeError("errMsg should be a Column or a str, got {}".format(type(errMsg))) @@ -5017,6 +5144,28 @@ def upper(col: "ColumnOrName") -> Column: Converts a string expression to upper case. .. versionadded:: 1.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + upper case values. + + Examples + -------- + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select(upper("value")).show() + +------------+ + |upper(value)| + +------------+ + | SPARK| + | PYSPARK| + | PANDAS API| + +------------+ """ return _invoke_function_over_columns("upper", col) @@ -5026,6 +5175,28 @@ def lower(col: "ColumnOrName") -> Column: Converts a string expression to lower case. .. versionadded:: 1.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + lower case values. + + Examples + -------- + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select(lower("value")).show() + +------------+ + |lower(value)| + +------------+ + | spark| + | pyspark| + | pandas api| + +------------+ """ return _invoke_function_over_columns("lower", col) @@ -5035,6 +5206,28 @@ def ascii(col: "ColumnOrName") -> Column: Computes the numeric value of the first character of the string column. .. versionadded:: 1.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + numeric value. + + Examples + -------- + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select(ascii("value")).show() + +------------+ + |ascii(value)| + +------------+ + | 83| + | 80| + | 80| + +------------+ """ return _invoke_function_over_columns("ascii", col) @@ -5044,6 +5237,28 @@ def base64(col: "ColumnOrName") -> Column: Computes the BASE64 encoding of a binary column and returns it as a string column. .. versionadded:: 1.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + BASE64 encoding of string value. + + Examples + -------- + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select(base64("value")).show() + +----------------+ + | base64(value)| + +----------------+ + | U3Bhcms=| + | UHlTcGFyaw==| + |UGFuZGFzIEFQSQ==| + +----------------+ """ return _invoke_function_over_columns("base64", col) @@ -5053,6 +5268,30 @@ def unbase64(col: "ColumnOrName") -> Column: Decodes a BASE64 encoded string column and returns it as a binary column. .. versionadded:: 1.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + encoded string value. + + Examples + -------- + >>> df = spark.createDataFrame(["U3Bhcms=", + ... "UHlTcGFyaw==", + ... "UGFuZGFzIEFQSQ=="], "STRING") + >>> df.select(unbase64("value")).show() + +--------------------+ + | unbase64(value)| + +--------------------+ + | [53 70 61 72 6B]| + |[50 79 53 70 61 7...| + |[50 61 6E 64 61 7...| + +--------------------+ """ return _invoke_function_over_columns("unbase64", col) @@ -5062,6 +5301,28 @@ def ltrim(col: "ColumnOrName") -> Column: Trim the spaces from left end for the specified string value. .. versionadded:: 1.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + left trimmed values. + + Examples + -------- + >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") + >>> df.select(ltrim("value").alias("r")).withColumn("length", length("r")).show() + +-------+------+ + | r|length| + +-------+------+ + | Spark| 5| + |Spark | 7| + | Spark| 5| + +-------+------+ """ return _invoke_function_over_columns("ltrim", col) @@ -5071,6 +5332,28 @@ def rtrim(col: "ColumnOrName") -> Column: Trim the spaces from right end for the specified string value. .. versionadded:: 1.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + right trimmed values. + + Examples + -------- + >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") + >>> df.select(rtrim("value").alias("r")).withColumn("length", length("r")).show() + +--------+------+ + | r|length| + +--------+------+ + | Spark| 8| + | Spark| 5| + | Spark| 6| + +--------+------+ """ return _invoke_function_over_columns("rtrim", col) @@ -5080,6 +5363,28 @@ def trim(col: "ColumnOrName") -> Column: Trim the spaces from both ends for the specified string column. .. versionadded:: 1.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + trimmed values from both sides. + + Examples + -------- + >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") + >>> df.select(trim("value").alias("r")).withColumn("length", length("r")).show() + +-----+------+ + | r|length| + +-----+------+ + |Spark| 5| + |Spark| 5| + |Spark| 5| + +-----+------+ """ return _invoke_function_over_columns("trim", col) @@ -5091,6 +5396,18 @@ def concat_ws(sep: str, *cols: "ColumnOrName") -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + sep : str + words seperator. + cols : :class:`~pyspark.sql.Column` or str + list of columns to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + string of concatenated words. + Examples -------- >>> df = spark.createDataFrame([('abcd','123')], ['s', 'd']) @@ -5108,6 +5425,28 @@ def decode(col: "ColumnOrName", charset: str) -> Column: (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'). .. versionadded:: 1.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + charset : str + charset to use to decode to. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. + + Examples + -------- + >>> df = spark.createDataFrame([('abcd',)], ['a']) + >>> df.select(decode("a", "UTF-8")).show() + +----------------------+ + |stringdecode(a, UTF-8)| + +----------------------+ + | abcd| + +----------------------+ """ return _invoke_function("decode", _to_java_column(col), charset) @@ -5118,6 +5457,28 @@ def encode(col: "ColumnOrName", charset: str) -> Column: (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'). .. versionadded:: 1.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + charset : str + charset to use to encode. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. + + Examples + -------- + >>> df = spark.createDataFrame([('abcd',)], ['c']) + >>> df.select(encode("c", "UTF-8")).show() + +----------------+ + |encode(c, UTF-8)| + +----------------+ + | [61 62 63 64]| + +----------------+ """ return _invoke_function("encode", _to_java_column(col), charset) @@ -5136,6 +5497,11 @@ def format_number(col: "ColumnOrName", d: int) -> Column: d : int the N decimal places + Returns + ------- + :class:`~pyspark.sql.Column` + the column of formatted results. + >>> spark.createDataFrame([(5,)], ['a']).select(format_number('a', 4).alias('v')).collect() [Row(v='5.0000')] """ @@ -5155,6 +5521,11 @@ def format_string(format: str, *cols: "ColumnOrName") -> Column: cols : :class:`~pyspark.sql.Column` or str column names or :class:`~pyspark.sql.Column`\\s to be used in formatting + Returns + ------- + :class:`~pyspark.sql.Column` + the column of formatted results. + Examples -------- >>> df = spark.createDataFrame([(5, "hello")], ['a', 'b']) @@ -5178,6 +5549,20 @@ def instr(str: "ColumnOrName", substr: str) -> Column: The position is not zero based, but 1 based index. Returns 0 if substr could not be found in str. + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + target column to work on. + substr : str + substring to look for. + + Returns + ------- + :class:`~pyspark.sql.Column` + location of the first occurence of the substring as integer. + + Examples + -------- >>> df = spark.createDataFrame([('abcd',)], ['s',]) >>> df.select(instr(df.s, 'b').alias('s')).collect() [Row(s=2)] @@ -5205,10 +5590,15 @@ def overlay( column name or column containing the substitution string pos : :class:`~pyspark.sql.Column` or str or int column name, column, or int containing the starting position in src - len : :class:`~pyspark.sql.Column` or str or int + len : :class:`~pyspark.sql.Column` or str or int, optional column name, column, or int containing the number of bytes to replace in src string by 'replace' defaults to -1, which represents the length of the 'replace' string + Returns + ------- + :class:`~pyspark.sql.Column` + string with replaced values. + Examples -------- >>> df = spark.createDataFrame([("SPARK_SQL", "CORE")], ("x", "y")) @@ -5254,6 +5644,11 @@ def sentences( country : :class:`~pyspark.sql.Column` or str, optional a country of the locale + Returns + ------- + :class:`~pyspark.sql.Column` + arrays of split sentences. + Examples -------- >>> df = spark.createDataFrame([["This is an example sentence."]], ["string"]) @@ -5263,6 +5658,13 @@ def sentences( +-----------------------------------+ |[[This, is, an, example, sentence]]| +-----------------------------------+ + >>> df = spark.createDataFrame([["Hello world. How are you?"]], ["s"]) + >>> df.select(sentences("s")).show(truncate=False) + +---------------------------------+ + |sentences(s, , ) | + +---------------------------------+ + |[[Hello, world], [How, are, you]]| + +---------------------------------+ """ if language is None: language = lit("") @@ -5284,6 +5686,20 @@ def substring(str: "ColumnOrName", pos: int, len: int) -> Column: ----- The position is not zero based, but 1 based index. + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + target column to work on. + pos : int + starting position in str. + len : int + length of chars. + + Returns + ------- + :class:`~pyspark.sql.Column` + substring of given value. + Examples -------- >>> df = spark.createDataFrame([('abcd',)], ['s',]) @@ -5302,6 +5718,20 @@ def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + target column to work on. + delim : str + delimiter of values. + count : int + number of occurences. + + Returns + ------- + :class:`~pyspark.sql.Column` + substring of given value. + Examples -------- >>> df = spark.createDataFrame([('a.b.c.d',)], ['s']) @@ -5318,6 +5748,18 @@ def levenshtein(left: "ColumnOrName", right: "ColumnOrName") -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + left : :class:`~pyspark.sql.Column` or str + first column value. + right : :class:`~pyspark.sql.Column` or str + second column value. + + Returns + ------- + :class:`~pyspark.sql.Column` + Levenshtein distance as integer value. + Examples -------- >>> df0 = spark.createDataFrame([('kitten', 'sitting',)], ['l', 'r']) @@ -5342,6 +5784,11 @@ def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column: pos : int, optional start position (zero based) + Returns + ------- + :class:`~pyspark.sql.Column` + position of the substring. + Notes ----- The position is not zero based, but 1 based index. Returns 0 if substr @@ -5362,6 +5809,20 @@ def lpad(col: "ColumnOrName", len: int, pad: str) -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + len : int + length of the final string. + pad : str + chars to prepend. + + Returns + ------- + :class:`~pyspark.sql.Column` + left padded result. + Examples -------- >>> df = spark.createDataFrame([('abcd',)], ['s',]) @@ -5377,6 +5838,20 @@ def rpad(col: "ColumnOrName", len: int, pad: str) -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + len : int + length of the final string. + pad : str + chars to append. + + Returns + ------- + :class:`~pyspark.sql.Column` + right padded result. + Examples -------- >>> df = spark.createDataFrame([('abcd',)], ['s',]) @@ -5392,6 +5867,18 @@ def repeat(col: "ColumnOrName", n: int) -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + n : int + number of times to repeat value. + + Returns + ------- + :class:`~pyspark.sql.Column` + string with repeated values. + Examples -------- >>> df = spark.createDataFrame([('ab',)], ['s',]) @@ -5426,6 +5913,11 @@ def split(str: "ColumnOrName", pattern: str, limit: int = -1) -> Column: .. versionchanged:: 3.0 `split` now takes an optional `limit` field. If not provided, default limit value is -1. + Returns + ------- + :class:`~pyspark.sql.Column` + array of separated strings. + Examples -------- >>> df = spark.createDataFrame([('oneAtwoBthreeC',)], ['s',]) @@ -5443,6 +5935,20 @@ def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + target column to work on. + pattern : str + regex pattern to apply. + idx : int + matched group id. + + Returns + ------- + :class:`~pyspark.sql.Column` + matched value specified by `idx` group id. + Examples -------- >>> df = spark.createDataFrame([('100-200',)], ['str']) @@ -5474,6 +5980,11 @@ def regexp_replace( replacement : :class:`~pyspark.sql.Column` or str column object or str containing the replacement + Returns + ------- + :class:`~pyspark.sql.Column` + string with all substrings replaced. + Examples -------- >>> df = spark.createDataFrame([("100-200", r"(\d+)", "--")], ["str", "pattern", "replacement"]) @@ -5498,6 +6009,16 @@ def initcap(col: "ColumnOrName") -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + string with all first letters are uppercase in each word. + Examples -------- >>> spark.createDataFrame([('ab cd',)], ['a']).select(initcap("a").alias('v')).collect() @@ -5512,6 +6033,16 @@ def soundex(col: "ColumnOrName") -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + SoundEx encoded string. + Examples -------- >>> df = spark.createDataFrame([("Peters",),("Uhrbach",)], ['name']) @@ -5526,6 +6057,16 @@ def bin(col: "ColumnOrName") -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + binary representation of given value as string. + Examples -------- >>> df.select(bin(df.age).alias('c')).collect() @@ -5541,6 +6082,16 @@ def hex(col: "ColumnOrName") -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + hexadecimal representation of given value as string. + Examples -------- >>> spark.createDataFrame([('ABC', 3)], ['a', 'b']).select(hex('a'), hex('b')).collect() @@ -5555,6 +6106,16 @@ def unhex(col: "ColumnOrName") -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + string representation of given hexadecimal value. + Examples -------- >>> spark.createDataFrame([('414243',)], ['a']).select(unhex('a')).collect() @@ -5570,6 +6131,16 @@ def length(col: "ColumnOrName") -> Column: .. versionadded:: 1.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + target column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + length of the value. + Examples -------- >>> spark.createDataFrame([('ABC ',)], ['a']).select(length('a').alias('length')).collect() @@ -5633,11 +6204,26 @@ def bit_length(col: "ColumnOrName") -> Column: def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column: """A function translate any character in the `srcCol` by a character in `matching`. The characters in `replace` is corresponding to the characters in `matching`. - The translate will happen when any character in the string matching with the character + Translation will happen whenever any character in the string is matching with the character in the `matching`. .. versionadded:: 1.5.0 + Parameters + ---------- + srcCol : :class:`~pyspark.sql.Column` or str + Source column or strings + matching : str + matching characters. + replace : str + characters for replacement. If this is shorter than `matching` string then + those chars that don't have replacement will be dropped. + + Returns + ------- + :class:`~pyspark.sql.Column` + replaced value. + Examples -------- >>> spark.createDataFrame([('translate',)], ['a']).select(translate('a', "rnlt", "123") \\ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org