LuciferYang commented on code in PR #44747:
URL: https://github.com/apache/spark/pull/44747#discussion_r1452940467
##########
python/pyspark/sql/functions/builtin.py:
##########
@@ -17181,16 +17183,39 @@ def map_filter(col: "ColumnOrName", f:
Callable[[Column, Column], Column]) -> Co
Returns
-------
:class:`~pyspark.sql.Column`
- filtered map.
+ A new map column containing only the key-value pairs that satisfy the
predicate.
Examples
--------
+ Example 1: Filtering a map with a simple condition
+
+ >>> from pyspark.sql import functions as sf
>>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz":
32.0})], ("id", "data"))
- >>> row = df.select(map_filter(
- ... "data", lambda _, v: v > 30.0).alias("data_filtered")
+ >>> row = df.select(
+ ... sf.map_filter("data", lambda _, v: v > 30.0).alias("data_filtered")
... ).head()
>>> sorted(row["data_filtered"].items())
[('baz', 32.0), ('foo', 42.0)]
+
+ Example 2: Filtering a map with a condition on keys
+
+ >>> from pyspark.sql import functions as sf
+ >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz":
32.0})], ("id", "data"))
+ >>> row = df.select(
+ ... sf.map_filter("data", lambda k, _:
k.startswith("b")).alias("data_filtered")
+ ... ).head()
+ >>> sorted(row["data_filtered"].items())
+ [('bar', 1.0), ('baz', 32.0)]
Review Comment:
Not use the 'show' because the result data sorting is not stable
##########
python/pyspark/sql/functions/builtin.py:
##########
@@ -17225,20 +17253,50 @@ def map_zip_with(
Returns
-------
:class:`~pyspark.sql.Column`
- zipped map where entries are calculated by applying given function to
each
- pair of arguments.
+ A new map column where each key-value pair is the result of applying
the function to
+ the corresponding key-value pairs in the input maps.
Examples
--------
+ Example 1: Merging two maps with a simple function
+
+ >>> from pyspark.sql import functions as sf
>>> df = spark.createDataFrame([
- ... (1, {"IT": 24.0, "SALES": 12.00}, {"IT": 2.0, "SALES": 1.4})],
- ... ("id", "base", "ratio")
- ... )
- >>> row = df.select(map_zip_with(
- ... "base", "ratio", lambda k, v1, v2: round(v1 * v2,
2)).alias("updated_data")
+ ... (1, {"A": 1, "B": 2}, {"A": 3, "B": 4})],
+ ... ("id", "map1", "map2"))
+ >>> row = df.select(
+ ... sf.map_zip_with("map1", "map2", lambda _, v1, v2: v1 +
v2).alias("updated_data")
+ ... ).head()
+ >>> sorted(row["updated_data"].items())
+ [('A', 4), ('B', 6)]
+
+ Example 2: Merging two maps with a complex function
+
+ >>> from pyspark.sql import functions as sf
+ >>> df = spark.createDataFrame([
+ ... (1, {"A": 1, "B": 2}, {"A": 3, "B": 4})],
+ ... ("id", "map1", "map2"))
+ >>> row = df.select(
+ ... sf.map_zip_with("map1", "map2",
+ ... lambda k, v1, v2: sf.when(k == "A", v1 + v2).otherwise(v1 - v2)
+ ... ).alias("updated_data")
+ ... ).head()
+ >>> sorted(row["updated_data"].items())
+ [('A', 4), ('B', -2)]
Review Comment:
Not use the 'show' because the result data sorting is not stable
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
