HyukjinKwon commented on a change in pull request #30143:
URL: https://github.com/apache/spark/pull/30143#discussion_r512367831
##########
File path: python/pyspark/sql/functions.py
##########
@@ -42,154 +41,457 @@
# since it requires to make every single overridden definition.
-def _create_function(name, doc=""):
- """Create a PySpark function by its name"""
- def _(col):
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)(col._jc if isinstance(col,
Column) else col)
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
+def _get_get_jvm_function(name, sc):
+ """
+ Retrieves JVM function identified by name from
+ Java gateway associated with sc.
+ """
+ return getattr(sc._jvm.functions, name)
-def _create_function_over_column(name, doc=""):
- """Similar with `_create_function` but creates a PySpark function that
takes a column
- (as string as well). This is mainly for PySpark functions to take strings
as
- column names.
+def _invoke_function(name, *args):
+ """
+ Invokes JVM function identified by name with args
+ and wraps the result with :class:`Column`.
"""
- def _(col):
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)(_to_java_column(col))
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
+ jf = _get_get_jvm_function(name, SparkContext._active_spark_context)
+ return Column(jf(*args))
-def _wrap_deprecated_function(func, message):
- """ Wrap the deprecated function to print out deprecation warnings"""
- def _(col):
- warnings.warn(message, DeprecationWarning)
- return func(col)
- return functools.wraps(func)(_)
+def _invoke_function_over_column(name, col):
+ """
+ Invokes unary JVM function identified by name
+ and wraps the result with :class:`Column`.
+ """
+ return _invoke_function(name, _to_java_column(col))
-def _create_binary_mathfunction(name, doc=""):
- """ Create a binary mathfunction by name"""
- def _(col1, col2):
- sc = SparkContext._active_spark_context
+def _invoke_binary_math_function(name, col1, col2):
+ """
+ Invokes binary JVM math function identified by name
+ and wraps the result with :class:`Column`.
+ """
+ return _invoke_function(
+ name,
# For legacy reasons, the arguments here can be implicitly converted
into floats,
# if they are not columns or strings.
- if isinstance(col1, Column):
- arg1 = col1._jc
- elif isinstance(col1, str):
- arg1 = _create_column_from_name(col1)
- else:
- arg1 = float(col1)
-
- if isinstance(col2, Column):
- arg2 = col2._jc
- elif isinstance(col2, str):
- arg2 = _create_column_from_name(col2)
- else:
- arg2 = float(col2)
-
- jc = getattr(sc._jvm.functions, name)(arg1, arg2)
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
-
-
-def _create_window_function(name, doc=''):
- """ Create a window function by name """
- def _():
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)()
- return Column(jc)
- _.__name__ = name
- _.__doc__ = 'Window function: ' + doc
- return _
+ _to_java_column(col1) if isinstance(col1, (str, Column)) else
float(col1),
+ _to_java_column(col2) if isinstance(col2, (str, Column)) else
float(col2)
+ )
def _options_to_str(options):
return {key: to_str(value) for (key, value) in options.items()}
-_lit_doc = """
+
+@since(1.3)
+def lit(col):
+ """
Creates a :class:`Column` of literal value.
>>> df.select(lit(5).alias('height')).withColumn('spark_user',
lit(True)).take(1)
[Row(height=5, spark_user=True)]
"""
-_functions = {
- 'lit': _lit_doc,
- 'col': 'Returns a :class:`Column` based on the given column name.',
- 'column': 'Returns a :class:`Column` based on the given column name.',
- 'asc': 'Returns a sort expression based on the ascending order of the
given column name.',
- 'desc': 'Returns a sort expression based on the descending order of the
given column name.',
-}
-
-_functions_over_column = {
- 'sqrt': 'Computes the square root of the specified float value.',
- 'abs': 'Computes the absolute value.',
-
- 'max': 'Aggregate function: returns the maximum value of the expression in
a group.',
- 'min': 'Aggregate function: returns the minimum value of the expression in
a group.',
- 'count': 'Aggregate function: returns the number of items in a group.',
- 'sum': 'Aggregate function: returns the sum of all values in the
expression.',
- 'avg': 'Aggregate function: returns the average of the values in a group.',
- 'mean': 'Aggregate function: returns the average of the values in a
group.',
- 'sumDistinct': 'Aggregate function: returns the sum of distinct values in
the expression.',
-}
-
-_functions_1_4_over_column = {
- # unary math functions
- 'acos': ':return: inverse cosine of `col`, as if computed by
`java.lang.Math.acos()`',
- 'asin': ':return: inverse sine of `col`, as if computed by
`java.lang.Math.asin()`',
- 'atan': ':return: inverse tangent of `col`, as if computed by
`java.lang.Math.atan()`',
- 'cbrt': 'Computes the cube-root of the given value.',
- 'ceil': 'Computes the ceiling of the given value.',
- 'cos': """:param col: angle in radians
- :return: cosine of the angle, as if computed by
`java.lang.Math.cos()`.""",
- 'cosh': """:param col: hyperbolic angle
- :return: hyperbolic cosine of the angle, as if computed by
`java.lang.Math.cosh()`""",
- 'exp': 'Computes the exponential of the given value.',
- 'expm1': 'Computes the exponential of the given value minus one.',
- 'floor': 'Computes the floor of the given value.',
- 'log': 'Computes the natural logarithm of the given value.',
- 'log10': 'Computes the logarithm of the given value in Base 10.',
- 'log1p': 'Computes the natural logarithm of the given value plus one.',
- 'rint': 'Returns the double value that is closest in value to the argument
and' +
- ' is equal to a mathematical integer.',
- 'signum': 'Computes the signum of the given value.',
- 'sin': """:param col: angle in radians
- :return: sine of the angle, as if computed by
`java.lang.Math.sin()`""",
- 'sinh': """:param col: hyperbolic angle
- :return: hyperbolic sine of the given value,
- as if computed by `java.lang.Math.sinh()`""",
- 'tan': """:param col: angle in radians
- :return: tangent of the given value, as if computed by
`java.lang.Math.tan()`""",
- 'tanh': """:param col: hyperbolic angle
- :return: hyperbolic tangent of the given value,
- as if computed by `java.lang.Math.tanh()`""",
- 'toDegrees': '.. note:: Deprecated in 2.1, use :func:`degrees` instead.',
- 'toRadians': '.. note:: Deprecated in 2.1, use :func:`radians` instead.',
- 'bitwiseNOT': 'Computes bitwise not.',
-}
-
-_functions_2_4 = {
- 'asc_nulls_first': 'Returns a sort expression based on the ascending order
of the given' +
- ' column name, and null values return before non-null
values.',
- 'asc_nulls_last': 'Returns a sort expression based on the ascending order
of the given' +
- ' column name, and null values appear after non-null
values.',
- 'desc_nulls_first': 'Returns a sort expression based on the descending
order of the given' +
- ' column name, and null values appear before non-null
values.',
- 'desc_nulls_last': 'Returns a sort expression based on the descending
order of the given' +
- ' column name, and null values appear after non-null
values',
-}
-
-_collect_list_doc = """
+ return col if isinstance(col, Column) else _invoke_function("lit", col)
+
+
+@since(1.3)
+def col(col):
+ """
+ Returns a :class:`Column` based on the given column name.'
+ """
+ return _invoke_function("col", col)
+
+
+@since(1.3)
+def column(col):
+ """
+ Returns a :class:`Column` based on the given column name.'
+ """
+ return col(col)
+
+
+@since(1.3)
+def asc(col):
+ """
+ Returns a sort expression based on the ascending order of the given column
name.
+ """
+ return _invoke_function("asc", col)
+
+
+@since(1.3)
+def desc(col):
+ """
+ Returns a sort expression based on the descending order of the given
column name.
+ """
+ return _invoke_function("desc", col)
Review comment:
This one would have to be same as `asc` above.
##########
File path: python/pyspark/sql/functions.py
##########
@@ -42,154 +41,457 @@
# since it requires to make every single overridden definition.
-def _create_function(name, doc=""):
- """Create a PySpark function by its name"""
- def _(col):
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)(col._jc if isinstance(col,
Column) else col)
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
+def _get_get_jvm_function(name, sc):
+ """
+ Retrieves JVM function identified by name from
+ Java gateway associated with sc.
+ """
+ return getattr(sc._jvm.functions, name)
-def _create_function_over_column(name, doc=""):
- """Similar with `_create_function` but creates a PySpark function that
takes a column
- (as string as well). This is mainly for PySpark functions to take strings
as
- column names.
+def _invoke_function(name, *args):
+ """
+ Invokes JVM function identified by name with args
+ and wraps the result with :class:`Column`.
"""
- def _(col):
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)(_to_java_column(col))
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
+ jf = _get_get_jvm_function(name, SparkContext._active_spark_context)
+ return Column(jf(*args))
-def _wrap_deprecated_function(func, message):
- """ Wrap the deprecated function to print out deprecation warnings"""
- def _(col):
- warnings.warn(message, DeprecationWarning)
- return func(col)
- return functools.wraps(func)(_)
+def _invoke_function_over_column(name, col):
+ """
+ Invokes unary JVM function identified by name
+ and wraps the result with :class:`Column`.
+ """
+ return _invoke_function(name, _to_java_column(col))
-def _create_binary_mathfunction(name, doc=""):
- """ Create a binary mathfunction by name"""
- def _(col1, col2):
- sc = SparkContext._active_spark_context
+def _invoke_binary_math_function(name, col1, col2):
+ """
+ Invokes binary JVM math function identified by name
+ and wraps the result with :class:`Column`.
+ """
+ return _invoke_function(
+ name,
# For legacy reasons, the arguments here can be implicitly converted
into floats,
# if they are not columns or strings.
- if isinstance(col1, Column):
- arg1 = col1._jc
- elif isinstance(col1, str):
- arg1 = _create_column_from_name(col1)
- else:
- arg1 = float(col1)
-
- if isinstance(col2, Column):
- arg2 = col2._jc
- elif isinstance(col2, str):
- arg2 = _create_column_from_name(col2)
- else:
- arg2 = float(col2)
-
- jc = getattr(sc._jvm.functions, name)(arg1, arg2)
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
-
-
-def _create_window_function(name, doc=''):
- """ Create a window function by name """
- def _():
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)()
- return Column(jc)
- _.__name__ = name
- _.__doc__ = 'Window function: ' + doc
- return _
+ _to_java_column(col1) if isinstance(col1, (str, Column)) else
float(col1),
+ _to_java_column(col2) if isinstance(col2, (str, Column)) else
float(col2)
+ )
def _options_to_str(options):
return {key: to_str(value) for (key, value) in options.items()}
-_lit_doc = """
+
+@since(1.3)
+def lit(col):
+ """
Creates a :class:`Column` of literal value.
>>> df.select(lit(5).alias('height')).withColumn('spark_user',
lit(True)).take(1)
[Row(height=5, spark_user=True)]
"""
-_functions = {
- 'lit': _lit_doc,
- 'col': 'Returns a :class:`Column` based on the given column name.',
- 'column': 'Returns a :class:`Column` based on the given column name.',
- 'asc': 'Returns a sort expression based on the ascending order of the
given column name.',
- 'desc': 'Returns a sort expression based on the descending order of the
given column name.',
-}
-
-_functions_over_column = {
- 'sqrt': 'Computes the square root of the specified float value.',
- 'abs': 'Computes the absolute value.',
-
- 'max': 'Aggregate function: returns the maximum value of the expression in
a group.',
- 'min': 'Aggregate function: returns the minimum value of the expression in
a group.',
- 'count': 'Aggregate function: returns the number of items in a group.',
- 'sum': 'Aggregate function: returns the sum of all values in the
expression.',
- 'avg': 'Aggregate function: returns the average of the values in a group.',
- 'mean': 'Aggregate function: returns the average of the values in a
group.',
- 'sumDistinct': 'Aggregate function: returns the sum of distinct values in
the expression.',
-}
-
-_functions_1_4_over_column = {
- # unary math functions
- 'acos': ':return: inverse cosine of `col`, as if computed by
`java.lang.Math.acos()`',
- 'asin': ':return: inverse sine of `col`, as if computed by
`java.lang.Math.asin()`',
- 'atan': ':return: inverse tangent of `col`, as if computed by
`java.lang.Math.atan()`',
- 'cbrt': 'Computes the cube-root of the given value.',
- 'ceil': 'Computes the ceiling of the given value.',
- 'cos': """:param col: angle in radians
- :return: cosine of the angle, as if computed by
`java.lang.Math.cos()`.""",
- 'cosh': """:param col: hyperbolic angle
- :return: hyperbolic cosine of the angle, as if computed by
`java.lang.Math.cosh()`""",
- 'exp': 'Computes the exponential of the given value.',
- 'expm1': 'Computes the exponential of the given value minus one.',
- 'floor': 'Computes the floor of the given value.',
- 'log': 'Computes the natural logarithm of the given value.',
- 'log10': 'Computes the logarithm of the given value in Base 10.',
- 'log1p': 'Computes the natural logarithm of the given value plus one.',
- 'rint': 'Returns the double value that is closest in value to the argument
and' +
- ' is equal to a mathematical integer.',
- 'signum': 'Computes the signum of the given value.',
- 'sin': """:param col: angle in radians
- :return: sine of the angle, as if computed by
`java.lang.Math.sin()`""",
- 'sinh': """:param col: hyperbolic angle
- :return: hyperbolic sine of the given value,
- as if computed by `java.lang.Math.sinh()`""",
- 'tan': """:param col: angle in radians
- :return: tangent of the given value, as if computed by
`java.lang.Math.tan()`""",
- 'tanh': """:param col: hyperbolic angle
- :return: hyperbolic tangent of the given value,
- as if computed by `java.lang.Math.tanh()`""",
- 'toDegrees': '.. note:: Deprecated in 2.1, use :func:`degrees` instead.',
- 'toRadians': '.. note:: Deprecated in 2.1, use :func:`radians` instead.',
- 'bitwiseNOT': 'Computes bitwise not.',
-}
-
-_functions_2_4 = {
- 'asc_nulls_first': 'Returns a sort expression based on the ascending order
of the given' +
- ' column name, and null values return before non-null
values.',
- 'asc_nulls_last': 'Returns a sort expression based on the ascending order
of the given' +
- ' column name, and null values appear after non-null
values.',
- 'desc_nulls_first': 'Returns a sort expression based on the descending
order of the given' +
- ' column name, and null values appear before non-null
values.',
- 'desc_nulls_last': 'Returns a sort expression based on the descending
order of the given' +
- ' column name, and null values appear after non-null
values',
-}
-
-_collect_list_doc = """
+ return col if isinstance(col, Column) else _invoke_function("lit", col)
+
+
+@since(1.3)
+def col(col):
+ """
+ Returns a :class:`Column` based on the given column name.'
+ """
+ return _invoke_function("col", col)
+
+
+@since(1.3)
+def column(col):
+ """
+ Returns a :class:`Column` based on the given column name.'
+ """
+ return col(col)
+
+
+@since(1.3)
+def asc(col):
+ """
+ Returns a sort expression based on the ascending order of the given column
name.
+ """
+ return _invoke_function("asc", col)
Review comment:
@zero323, looks like we should call `col.asc if isinstance(col, Column)
else _invoke_function("asc", col)` instead to make it support both string and
columns in PySpark side (Scala side does not have the Column signature).
It was decided to support both string/column instances where we can in
PySpark at SPARK-26979.
Ideally we should do this in a separate JIRA and PR but I don't mind fixing
it here while we're here. I will leave it to you.
##########
File path: python/pyspark/sql/functions.py
##########
@@ -42,154 +41,457 @@
# since it requires to make every single overridden definition.
-def _create_function(name, doc=""):
- """Create a PySpark function by its name"""
- def _(col):
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)(col._jc if isinstance(col,
Column) else col)
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
+def _get_get_jvm_function(name, sc):
+ """
+ Retrieves JVM function identified by name from
+ Java gateway associated with sc.
+ """
+ return getattr(sc._jvm.functions, name)
-def _create_function_over_column(name, doc=""):
- """Similar with `_create_function` but creates a PySpark function that
takes a column
- (as string as well). This is mainly for PySpark functions to take strings
as
- column names.
+def _invoke_function(name, *args):
+ """
+ Invokes JVM function identified by name with args
+ and wraps the result with :class:`Column`.
"""
- def _(col):
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)(_to_java_column(col))
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
+ jf = _get_get_jvm_function(name, SparkContext._active_spark_context)
+ return Column(jf(*args))
-def _wrap_deprecated_function(func, message):
- """ Wrap the deprecated function to print out deprecation warnings"""
- def _(col):
- warnings.warn(message, DeprecationWarning)
- return func(col)
- return functools.wraps(func)(_)
+def _invoke_function_over_column(name, col):
+ """
+ Invokes unary JVM function identified by name
+ and wraps the result with :class:`Column`.
+ """
+ return _invoke_function(name, _to_java_column(col))
-def _create_binary_mathfunction(name, doc=""):
- """ Create a binary mathfunction by name"""
- def _(col1, col2):
- sc = SparkContext._active_spark_context
+def _invoke_binary_math_function(name, col1, col2):
+ """
+ Invokes binary JVM math function identified by name
+ and wraps the result with :class:`Column`.
+ """
+ return _invoke_function(
+ name,
# For legacy reasons, the arguments here can be implicitly converted
into floats,
# if they are not columns or strings.
- if isinstance(col1, Column):
- arg1 = col1._jc
- elif isinstance(col1, str):
- arg1 = _create_column_from_name(col1)
- else:
- arg1 = float(col1)
-
- if isinstance(col2, Column):
- arg2 = col2._jc
- elif isinstance(col2, str):
- arg2 = _create_column_from_name(col2)
- else:
- arg2 = float(col2)
-
- jc = getattr(sc._jvm.functions, name)(arg1, arg2)
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
-
-
-def _create_window_function(name, doc=''):
- """ Create a window function by name """
- def _():
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)()
- return Column(jc)
- _.__name__ = name
- _.__doc__ = 'Window function: ' + doc
- return _
+ _to_java_column(col1) if isinstance(col1, (str, Column)) else
float(col1),
+ _to_java_column(col2) if isinstance(col2, (str, Column)) else
float(col2)
Review comment:
Much more neat!
##########
File path: python/pyspark/sql/functions.py
##########
@@ -42,154 +41,457 @@
# since it requires to make every single overridden definition.
-def _create_function(name, doc=""):
- """Create a PySpark function by its name"""
- def _(col):
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)(col._jc if isinstance(col,
Column) else col)
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
+def _get_get_jvm_function(name, sc):
+ """
+ Retrieves JVM function identified by name from
+ Java gateway associated with sc.
+ """
+ return getattr(sc._jvm.functions, name)
-def _create_function_over_column(name, doc=""):
- """Similar with `_create_function` but creates a PySpark function that
takes a column
- (as string as well). This is mainly for PySpark functions to take strings
as
- column names.
+def _invoke_function(name, *args):
+ """
+ Invokes JVM function identified by name with args
+ and wraps the result with :class:`Column`.
"""
- def _(col):
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)(_to_java_column(col))
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
+ jf = _get_get_jvm_function(name, SparkContext._active_spark_context)
+ return Column(jf(*args))
-def _wrap_deprecated_function(func, message):
- """ Wrap the deprecated function to print out deprecation warnings"""
- def _(col):
- warnings.warn(message, DeprecationWarning)
- return func(col)
- return functools.wraps(func)(_)
+def _invoke_function_over_column(name, col):
+ """
+ Invokes unary JVM function identified by name
+ and wraps the result with :class:`Column`.
+ """
+ return _invoke_function(name, _to_java_column(col))
-def _create_binary_mathfunction(name, doc=""):
- """ Create a binary mathfunction by name"""
- def _(col1, col2):
- sc = SparkContext._active_spark_context
+def _invoke_binary_math_function(name, col1, col2):
+ """
+ Invokes binary JVM math function identified by name
+ and wraps the result with :class:`Column`.
+ """
+ return _invoke_function(
+ name,
# For legacy reasons, the arguments here can be implicitly converted
into floats,
# if they are not columns or strings.
- if isinstance(col1, Column):
- arg1 = col1._jc
- elif isinstance(col1, str):
- arg1 = _create_column_from_name(col1)
- else:
- arg1 = float(col1)
-
- if isinstance(col2, Column):
- arg2 = col2._jc
- elif isinstance(col2, str):
- arg2 = _create_column_from_name(col2)
- else:
- arg2 = float(col2)
-
- jc = getattr(sc._jvm.functions, name)(arg1, arg2)
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
-
-
-def _create_window_function(name, doc=''):
- """ Create a window function by name """
- def _():
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)()
- return Column(jc)
- _.__name__ = name
- _.__doc__ = 'Window function: ' + doc
- return _
+ _to_java_column(col1) if isinstance(col1, (str, Column)) else
float(col1),
+ _to_java_column(col2) if isinstance(col2, (str, Column)) else
float(col2)
+ )
def _options_to_str(options):
return {key: to_str(value) for (key, value) in options.items()}
-_lit_doc = """
+
+@since(1.3)
+def lit(col):
+ """
Creates a :class:`Column` of literal value.
>>> df.select(lit(5).alias('height')).withColumn('spark_user',
lit(True)).take(1)
[Row(height=5, spark_user=True)]
"""
-_functions = {
- 'lit': _lit_doc,
- 'col': 'Returns a :class:`Column` based on the given column name.',
- 'column': 'Returns a :class:`Column` based on the given column name.',
- 'asc': 'Returns a sort expression based on the ascending order of the
given column name.',
- 'desc': 'Returns a sort expression based on the descending order of the
given column name.',
-}
-
-_functions_over_column = {
- 'sqrt': 'Computes the square root of the specified float value.',
- 'abs': 'Computes the absolute value.',
-
- 'max': 'Aggregate function: returns the maximum value of the expression in
a group.',
- 'min': 'Aggregate function: returns the minimum value of the expression in
a group.',
- 'count': 'Aggregate function: returns the number of items in a group.',
- 'sum': 'Aggregate function: returns the sum of all values in the
expression.',
- 'avg': 'Aggregate function: returns the average of the values in a group.',
- 'mean': 'Aggregate function: returns the average of the values in a
group.',
- 'sumDistinct': 'Aggregate function: returns the sum of distinct values in
the expression.',
-}
-
-_functions_1_4_over_column = {
- # unary math functions
- 'acos': ':return: inverse cosine of `col`, as if computed by
`java.lang.Math.acos()`',
- 'asin': ':return: inverse sine of `col`, as if computed by
`java.lang.Math.asin()`',
- 'atan': ':return: inverse tangent of `col`, as if computed by
`java.lang.Math.atan()`',
- 'cbrt': 'Computes the cube-root of the given value.',
- 'ceil': 'Computes the ceiling of the given value.',
- 'cos': """:param col: angle in radians
- :return: cosine of the angle, as if computed by
`java.lang.Math.cos()`.""",
- 'cosh': """:param col: hyperbolic angle
- :return: hyperbolic cosine of the angle, as if computed by
`java.lang.Math.cosh()`""",
- 'exp': 'Computes the exponential of the given value.',
- 'expm1': 'Computes the exponential of the given value minus one.',
- 'floor': 'Computes the floor of the given value.',
- 'log': 'Computes the natural logarithm of the given value.',
- 'log10': 'Computes the logarithm of the given value in Base 10.',
- 'log1p': 'Computes the natural logarithm of the given value plus one.',
- 'rint': 'Returns the double value that is closest in value to the argument
and' +
- ' is equal to a mathematical integer.',
- 'signum': 'Computes the signum of the given value.',
- 'sin': """:param col: angle in radians
- :return: sine of the angle, as if computed by
`java.lang.Math.sin()`""",
- 'sinh': """:param col: hyperbolic angle
- :return: hyperbolic sine of the given value,
- as if computed by `java.lang.Math.sinh()`""",
- 'tan': """:param col: angle in radians
- :return: tangent of the given value, as if computed by
`java.lang.Math.tan()`""",
- 'tanh': """:param col: hyperbolic angle
- :return: hyperbolic tangent of the given value,
- as if computed by `java.lang.Math.tanh()`""",
- 'toDegrees': '.. note:: Deprecated in 2.1, use :func:`degrees` instead.',
- 'toRadians': '.. note:: Deprecated in 2.1, use :func:`radians` instead.',
- 'bitwiseNOT': 'Computes bitwise not.',
-}
-
-_functions_2_4 = {
- 'asc_nulls_first': 'Returns a sort expression based on the ascending order
of the given' +
- ' column name, and null values return before non-null
values.',
- 'asc_nulls_last': 'Returns a sort expression based on the ascending order
of the given' +
- ' column name, and null values appear after non-null
values.',
- 'desc_nulls_first': 'Returns a sort expression based on the descending
order of the given' +
- ' column name, and null values appear before non-null
values.',
- 'desc_nulls_last': 'Returns a sort expression based on the descending
order of the given' +
- ' column name, and null values appear after non-null
values',
-}
-
-_collect_list_doc = """
+ return col if isinstance(col, Column) else _invoke_function("lit", col)
+
+
+@since(1.3)
+def col(col):
+ """
+ Returns a :class:`Column` based on the given column name.'
+ """
+ return _invoke_function("col", col)
+
+
+@since(1.3)
+def column(col):
+ """
+ Returns a :class:`Column` based on the given column name.'
+ """
+ return col(col)
+
+
+@since(1.3)
+def asc(col):
+ """
+ Returns a sort expression based on the ascending order of the given column
name.
+ """
+ return _invoke_function("asc", col)
+
+
+@since(1.3)
+def desc(col):
+ """
+ Returns a sort expression based on the descending order of the given
column name.
+ """
+ return _invoke_function("desc", col)
+
+
+@since(1.3)
+def sqrt(col):
+ """
+ Computes the square root of the specified float value.
+ """
+ return _invoke_function_over_column("sqrt", col)
+
+
+@since(1.3)
+def abs(col):
+ """
+ Computes the absolute value.
+ """
+ return _invoke_function_over_column("abs", col)
+
+
+@since(1.3)
+def max(col):
+ """
+ Aggregate function: returns the maximum value of the expression in a group.
+ """
+ return _invoke_function_over_column("max", col)
+
+
+@since(1.3)
+def min(col):
+ """
+ Aggregate function: returns the minimum value of the expression in a group.
+ """
+ return _invoke_function_over_column("min", col)
+
+
+@since(1.3)
+def count(col):
+ """
+ Aggregate function: returns the number of items in a group.
+ """
+ return _invoke_function_over_column("count", col)
+
+
+@since(1.3)
+def sum(col):
+ """
+ Aggregate function: returns the sum of all values in the expression.
+ """
+ return _invoke_function_over_column("sum", col)
+
+
+@since(1.3)
+def avg(col):
+ """
+ Aggregate function: returns the average of the values in a group.
+ """
+ return _invoke_function_over_column("avg", col)
+
+
+@since(1.3)
+def mean(col):
+ """
+ Aggregate function: returns the average of the values in a group.
+ """
+ return _invoke_function_over_column("mean", col)
+
+
+@since(1.3)
+def sumDistinct(col):
+ """
+ Aggregate function: returns the sum of distinct values in the expression.
+ """
+ return _invoke_function_over_column("sumDistinct", col)
+
+
+@since(1.4)
+def acos(col):
+ """
+ :return: inverse cosine of `col`, as if computed by `java.lang.Math.acos()`
+ """
+ return _invoke_function_over_column("acos", col)
+
+
+@since(1.4)
+def asin(col):
+ """
+ :return: inverse sine of `col`, as if computed by `java.lang.Math.asin()`
+ """
+ return _invoke_function_over_column("asin", col)
+
+
+@since(1.4)
+def atan(col):
+ """
+ :return: inverse tangent of `col`, as if computed by
`java.lang.Math.atan()`
+ """
+ return _invoke_function_over_column("atan", col)
+
+
+@since(1.4)
+def cbrt(col):
+ """
+ Computes the cube-root of the given value.
+ """
+ return _invoke_function_over_column("cbrt", col)
+
+
+@since(1.4)
+def ceil(col):
+ """
+ Computes the ceiling of the given value.
+ """
+ return _invoke_function_over_column("ceil", col)
+
+
+@since(1.4)
+def cos(col):
+ """
+ :param col: angle in radians
+ :return: cosine of the angle, as if computed by `java.lang.Math.cos()`.
+ """
+ return _invoke_function_over_column("cos", col)
+
+
+@since(1.4)
+def cosh(col):
+ """
+ :param col: hyperbolic angle
+ :return: hyperbolic cosine of the angle, as if computed by
`java.lang.Math.cosh()`
+ """
+ return _invoke_function_over_column("cosh", col)
+
+
+@since(1.4)
+def exp(col):
+ """
+ Computes the exponential of the given value.
+ """
+ return _invoke_function_over_column("exp", col)
+
+
+@since(1.4)
+def expm1(col):
+ """
+ Computes the exponential of the given value minus one.
+ """
+ return _invoke_function_over_column("expm1", col)
+
+
+@since(1.4)
+def floor(col):
+ """
+ Computes the floor of the given value.
+ """
+ return _invoke_function_over_column("floor", col)
+
+
+@since(1.4)
+def log(col):
+ """
+ Computes the natural logarithm of the given value.
+ """
+ return _invoke_function_over_column("log", col)
+
+
+@since(1.4)
+def log10(col):
+ """
+ Computes the logarithm of the given value in Base 10.
+ """
+ return _invoke_function_over_column("log10", col)
+
+
+@since(1.4)
+def log1p(col):
+ """
+ Computes the natural logarithm of the given value plus one.
+ """
+ return _invoke_function_over_column("log1p", col)
+
+
+@since(1.4)
+def rint(col):
+ """
+ Returns the double value that is closest in value to the argument and
+ is equal to a mathematical integer.
+ """
+ return _invoke_function_over_column("rint", col)
+
+
+@since(1.4)
+def signum(col):
+ """
+ Computes the signum of the given value.
+ """
+ return _invoke_function_over_column("signum", col)
+
+
+@since(1.4)
+def sin(col):
+ """
+ :param col: angle in radians
+ :return: sine of the angle, as if computed by `java.lang.Math.sin()`
+ """
+ return _invoke_function_over_column("sin", col)
+
+
+@since(1.4)
+def sinh(col):
+ """
+ :param col: hyperbolic angle
+ :return: hyperbolic sine of the given value,
+ as if computed by `java.lang.Math.sinh()`
+ """
+ return _invoke_function_over_column("sinh", col)
+
+
+@since(1.4)
+def tan(col):
+ """
+ :param col: angle in radians
+ :return: tangent of the given value, as if computed by
`java.lang.Math.tan()`
+ """
+ return _invoke_function_over_column("tan", col)
+
+
+@since(1.4)
+def tanh(col):
+ """
+ :param col: hyperbolic angle
+ :return: hyperbolic tangent of the given value
+ as if computed by `java.lang.Math.tanh()`
+ """
+ return _invoke_function_over_column("tanh", col)
+
+
+@since(1.4)
+def toDegrees(col):
+ """
+ .. note:: Deprecated in 2.1, use :func:`degrees` instead.
+ """
+ warnings.warn("Deprecated in 2.1, use degrees instead.",
DeprecationWarning)
+ return degrees(col)
+
+
+@since(1.4)
+def toRadians(col):
+ """
+ .. note:: Deprecated in 2.1, use :func:`radians` instead.
+ """
+ warnings.warn("Deprecated in 2.1, use radians instead.",
DeprecationWarning)
+ return radians(col)
+
+
+@since(1.4)
+def bitwiseNOT(col):
+ """
+ Computes bitwise not.
+ """
+ return _invoke_function_over_column("bitwiseNOT", col)
+
+
+@since(2.4)
+def asc_nulls_first(col):
+ """
+ Returns a sort expression based on the ascending order of the given
+ column name, and null values return before non-null values.
+ """
+ return _invoke_function("asc_nulls_first", col)
Review comment:
`asc_nulls_first`, `asc_nulls_last`, `desc_nulls_first` and
`desc_nulls_last` look the same case above.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
