aray commented on code in PR #37144:
URL: https://github.com/apache/spark/pull/37144#discussion_r917523520
##########
python/pyspark/sql/tests/test_functions.py:
##########
@@ -55,6 +55,62 @@
class FunctionsTests(ReusedSQLTestCase):
+ def test_function_parity(self):
+ from pyspark.sql import functions as py_functions
+ from inspect import getmembers, isfunction
+
+ # This test compares the available list of functions in
pyspark.sql.functions with those
+ # available in the Scala/Java DataFrame API in
org.apache.spark.sql.functions.
+ #
+ # NOTE FOR DEVELOPERS:
+ # If this test fails one of the following needs to happen
+ # * If a function was added to org.apache.spark.sql.functions it
either needs to be added to
+ # pyspark.sql.functions or added to the below
expected_missing_in_py set.
+ # * If a function was added to pyspark.sql.functions that was already
in
+ # org.apache.spark.sql.functions then it needs to be removed from
expected_missing_in_py
+ # below. If the function has a different name it needs to be added
to py_equiv_jvm
+ # mapping.
+ # * If it's not related to an added/removed function then likely the
exclusion list
+ # jvm_excluded_fn needs to be updated.
+
+ jvm_functions = self.sc._jvm.functions
+
+ jvm_fn_set = {name for (name, value) in getmembers(jvm_functions)}
+ py_fn_set = {
+ name for (name, value) in getmembers(py_functions, isfunction) if
name[0] != "_"
+ }
+
+ # Functions on the JVM side we do not expect to be available in python
because they are
+ # depreciated, irrelevant to python, or have equivalents.
+ jvm_excluded_fn = [
+ "callUDF", # depreciated, use call_udf
+ "typedlit", # Scala only
+ "typedLit", # Scala only
+ "monotonicallyIncreasingId", # depreciated, use
monotonically_increasing_id
+ "negate", # equivalent to python -expression
+ "not", # equivalent to python ~expression
+ ]
+
+ jvm_fn_set.difference_update(jvm_excluded_fn)
+
+ # For functions that are named differently in pyspark this is the
mapping of their
+ # python name to the JVM equivalent
+ py_equiv_jvm = {"create_map": "map"}
+ for py_name, jvm_name in py_equiv_jvm.items():
+ if py_name in py_fn_set:
+ py_fn_set.remove(py_name)
+ py_fn_set.add(jvm_name)
+
+ missing_in_py = jvm_fn_set.difference(py_fn_set)
+
+ # Functions that we expect to be missing in python until they are
added to pyspark
+ # TODO Add all of these functions to pyspark
Review Comment:
I've gone through the list and found existing or created new JIRA's for each
and updated TODO's as such.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
