zero323 commented on a change in pull request #30143:
URL: https://github.com/apache/spark/pull/30143#discussion_r512488156
##########
File path: python/pyspark/sql/functions.py
##########
@@ -42,154 +41,457 @@
# since it requires to make every single overridden definition.
-def _create_function(name, doc=""):
- """Create a PySpark function by its name"""
- def _(col):
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)(col._jc if isinstance(col,
Column) else col)
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
+def _get_get_jvm_function(name, sc):
+ """
+ Retrieves JVM function identified by name from
+ Java gateway associated with sc.
+ """
+ return getattr(sc._jvm.functions, name)
-def _create_function_over_column(name, doc=""):
- """Similar with `_create_function` but creates a PySpark function that
takes a column
- (as string as well). This is mainly for PySpark functions to take strings
as
- column names.
+def _invoke_function(name, *args):
+ """
+ Invokes JVM function identified by name with args
+ and wraps the result with :class:`Column`.
"""
- def _(col):
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)(_to_java_column(col))
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
+ jf = _get_get_jvm_function(name, SparkContext._active_spark_context)
+ return Column(jf(*args))
-def _wrap_deprecated_function(func, message):
- """ Wrap the deprecated function to print out deprecation warnings"""
- def _(col):
- warnings.warn(message, DeprecationWarning)
- return func(col)
- return functools.wraps(func)(_)
+def _invoke_function_over_column(name, col):
+ """
+ Invokes unary JVM function identified by name
+ and wraps the result with :class:`Column`.
+ """
+ return _invoke_function(name, _to_java_column(col))
-def _create_binary_mathfunction(name, doc=""):
- """ Create a binary mathfunction by name"""
- def _(col1, col2):
- sc = SparkContext._active_spark_context
+def _invoke_binary_math_function(name, col1, col2):
+ """
+ Invokes binary JVM math function identified by name
+ and wraps the result with :class:`Column`.
+ """
+ return _invoke_function(
+ name,
# For legacy reasons, the arguments here can be implicitly converted
into floats,
# if they are not columns or strings.
- if isinstance(col1, Column):
- arg1 = col1._jc
- elif isinstance(col1, str):
- arg1 = _create_column_from_name(col1)
- else:
- arg1 = float(col1)
-
- if isinstance(col2, Column):
- arg2 = col2._jc
- elif isinstance(col2, str):
- arg2 = _create_column_from_name(col2)
- else:
- arg2 = float(col2)
-
- jc = getattr(sc._jvm.functions, name)(arg1, arg2)
- return Column(jc)
- _.__name__ = name
- _.__doc__ = doc
- return _
-
-
-def _create_window_function(name, doc=''):
- """ Create a window function by name """
- def _():
- sc = SparkContext._active_spark_context
- jc = getattr(sc._jvm.functions, name)()
- return Column(jc)
- _.__name__ = name
- _.__doc__ = 'Window function: ' + doc
- return _
+ _to_java_column(col1) if isinstance(col1, (str, Column)) else
float(col1),
+ _to_java_column(col2) if isinstance(col2, (str, Column)) else
float(col2)
+ )
def _options_to_str(options):
return {key: to_str(value) for (key, value) in options.items()}
-_lit_doc = """
+
+@since(1.3)
+def lit(col):
+ """
Creates a :class:`Column` of literal value.
>>> df.select(lit(5).alias('height')).withColumn('spark_user',
lit(True)).take(1)
[Row(height=5, spark_user=True)]
"""
-_functions = {
- 'lit': _lit_doc,
- 'col': 'Returns a :class:`Column` based on the given column name.',
- 'column': 'Returns a :class:`Column` based on the given column name.',
- 'asc': 'Returns a sort expression based on the ascending order of the
given column name.',
- 'desc': 'Returns a sort expression based on the descending order of the
given column name.',
-}
-
-_functions_over_column = {
- 'sqrt': 'Computes the square root of the specified float value.',
- 'abs': 'Computes the absolute value.',
-
- 'max': 'Aggregate function: returns the maximum value of the expression in
a group.',
- 'min': 'Aggregate function: returns the minimum value of the expression in
a group.',
- 'count': 'Aggregate function: returns the number of items in a group.',
- 'sum': 'Aggregate function: returns the sum of all values in the
expression.',
- 'avg': 'Aggregate function: returns the average of the values in a group.',
- 'mean': 'Aggregate function: returns the average of the values in a
group.',
- 'sumDistinct': 'Aggregate function: returns the sum of distinct values in
the expression.',
-}
-
-_functions_1_4_over_column = {
- # unary math functions
- 'acos': ':return: inverse cosine of `col`, as if computed by
`java.lang.Math.acos()`',
- 'asin': ':return: inverse sine of `col`, as if computed by
`java.lang.Math.asin()`',
- 'atan': ':return: inverse tangent of `col`, as if computed by
`java.lang.Math.atan()`',
- 'cbrt': 'Computes the cube-root of the given value.',
- 'ceil': 'Computes the ceiling of the given value.',
- 'cos': """:param col: angle in radians
- :return: cosine of the angle, as if computed by
`java.lang.Math.cos()`.""",
- 'cosh': """:param col: hyperbolic angle
- :return: hyperbolic cosine of the angle, as if computed by
`java.lang.Math.cosh()`""",
- 'exp': 'Computes the exponential of the given value.',
- 'expm1': 'Computes the exponential of the given value minus one.',
- 'floor': 'Computes the floor of the given value.',
- 'log': 'Computes the natural logarithm of the given value.',
- 'log10': 'Computes the logarithm of the given value in Base 10.',
- 'log1p': 'Computes the natural logarithm of the given value plus one.',
- 'rint': 'Returns the double value that is closest in value to the argument
and' +
- ' is equal to a mathematical integer.',
- 'signum': 'Computes the signum of the given value.',
- 'sin': """:param col: angle in radians
- :return: sine of the angle, as if computed by
`java.lang.Math.sin()`""",
- 'sinh': """:param col: hyperbolic angle
- :return: hyperbolic sine of the given value,
- as if computed by `java.lang.Math.sinh()`""",
- 'tan': """:param col: angle in radians
- :return: tangent of the given value, as if computed by
`java.lang.Math.tan()`""",
- 'tanh': """:param col: hyperbolic angle
- :return: hyperbolic tangent of the given value,
- as if computed by `java.lang.Math.tanh()`""",
- 'toDegrees': '.. note:: Deprecated in 2.1, use :func:`degrees` instead.',
- 'toRadians': '.. note:: Deprecated in 2.1, use :func:`radians` instead.',
- 'bitwiseNOT': 'Computes bitwise not.',
-}
-
-_functions_2_4 = {
- 'asc_nulls_first': 'Returns a sort expression based on the ascending order
of the given' +
- ' column name, and null values return before non-null
values.',
- 'asc_nulls_last': 'Returns a sort expression based on the ascending order
of the given' +
- ' column name, and null values appear after non-null
values.',
- 'desc_nulls_first': 'Returns a sort expression based on the descending
order of the given' +
- ' column name, and null values appear before non-null
values.',
- 'desc_nulls_last': 'Returns a sort expression based on the descending
order of the given' +
- ' column name, and null values appear after non-null
values',
-}
-
-_collect_list_doc = """
+ return col if isinstance(col, Column) else _invoke_function("lit", col)
+
+
+@since(1.3)
+def col(col):
+ """
+ Returns a :class:`Column` based on the given column name.'
+ """
+ return _invoke_function("col", col)
+
+
+@since(1.3)
+def column(col):
+ """
+ Returns a :class:`Column` based on the given column name.'
+ """
+ return col(col)
+
+
+@since(1.3)
+def asc(col):
+ """
+ Returns a sort expression based on the ascending order of the given column
name.
+ """
+ return _invoke_function("asc", col)
Review comment:
I wonder if it makes more sense, for the sake of consistency ,to add
`Column` signatures on Scala side.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
