Repository: spark Updated Branches: refs/heads/master 66a792cd8 -> e5387018e
[SPARK-19975][PYTHON][SQL] Add map_keys and map_values functions to Python ## What changes were proposed in this pull request? This fix tries to address the issue in SPARK-19975 where we have `map_keys` and `map_values` functions in SQL yet there is no Python equivalent functions. This fix adds `map_keys` and `map_values` functions to Python. ## How was this patch tested? This fix is tested manually (See Python docs for examples). Author: Yong Tang <[email protected]> Closes #17328 from yongtang/SPARK-19975. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e5387018 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e5387018 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e5387018 Branch: refs/heads/master Commit: e5387018e76a9af1318e78c4133ee68232e6a159 Parents: 66a792c Author: Yong Tang <[email protected]> Authored: Mon Jun 19 11:40:07 2017 -0700 Committer: gatorsmile <[email protected]> Committed: Mon Jun 19 11:40:07 2017 -0700 ---------------------------------------------------------------------- python/pyspark/sql/functions.py | 40 ++++++++++++++++++++ .../scala/org/apache/spark/sql/functions.scala | 14 +++++++ 2 files changed, 54 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/e5387018/python/pyspark/sql/functions.py ---------------------------------------------------------------------- diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index d9b86af..240ae65 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -1855,6 +1855,46 @@ def sort_array(col, asc=True): return Column(sc._jvm.functions.sort_array(_to_java_column(col), asc)) +@since(2.3) +def map_keys(col): + """ + Collection function: Returns an unordered array containing the keys of the map. + + :param col: name of column or expression + + >>> from pyspark.sql.functions import map_keys + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(map_keys("data").alias("keys")).show() + +------+ + | keys| + +------+ + |[1, 2]| + +------+ + """ + sc = SparkContext._active_spark_context + return Column(sc._jvm.functions.map_keys(_to_java_column(col))) + + +@since(2.3) +def map_values(col): + """ + Collection function: Returns an unordered array containing the values of the map. + + :param col: name of column or expression + + >>> from pyspark.sql.functions import map_values + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(map_values("data").alias("values")).show() + +------+ + |values| + +------+ + |[a, b]| + +------+ + """ + sc = SparkContext._active_spark_context + return Column(sc._jvm.functions.map_values(_to_java_column(col))) + + # ---------------------------- User Defined Function ---------------------------------- def _wrap_function(sc, func, returnType): http://git-wip-us.apache.org/repos/asf/spark/blob/e5387018/sql/core/src/main/scala/org/apache/spark/sql/functions.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index 8d2e1f3..9a35a5c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -3161,6 +3161,20 @@ object functions { */ def sort_array(e: Column, asc: Boolean): Column = withExpr { SortArray(e.expr, lit(asc).expr) } + /** + * Returns an unordered array containing the keys of the map. + * @group collection_funcs + * @since 2.3.0 + */ + def map_keys(e: Column): Column = withExpr { MapKeys(e.expr) } + + /** + * Returns an unordered array containing the values of the map. + * @group collection_funcs + * @since 2.3.0 + */ + def map_values(e: Column): Column = withExpr { MapValues(e.expr) } + ////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////////// --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
