[
https://issues.apache.org/jira/browse/SPARK-41904?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sandeep Singh updated SPARK-41904:
----------------------------------
Description:
{code:java}
from pyspark.sql import Window
from pyspark.sql.functions import nth_value
df = self.spark.createDataFrame(
[
("a", 0, None),
("a", 1, "x"),
("a", 2, "y"),
("a", 3, "z"),
("a", 4, None),
("b", 1, None),
("b", 2, None),
],
schema=("key", "order", "value"),
)
w = Window.partitionBy("key").orderBy("order")
rs = df.select(
df.key,
df.order,
nth_value("value", 2).over(w),
nth_value("value", 2, False).over(w),
nth_value("value", 2, True).over(w),
).collect()
expected = [
("a", 0, None, None, None),
("a", 1, "x", "x", None),
("a", 2, "x", "x", "y"),
("a", 3, "x", "x", "y"),
("a", 4, "x", "x", "y"),
("b", 1, None, None, None),
("b", 2, None, None, None),
]
for r, ex in zip(sorted(rs), sorted(expected)):
self.assertEqual(tuple(r), ex[: len(r)]){code}
{code:java}
Traceback (most recent call last):
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/tests/test_functions.py",
line 755, in test_nth_value
self.assertEqual(tuple(r), ex[: len(r)])
AssertionError: Tuples differ: ('a', 1, 'x', None) != ('a', 1, 'x', 'x')
First differing element 3:
None
'x'
- ('a', 1, 'x', None)
? ^^^^
+ ('a', 1, 'x', 'x')
? ^^^
{code}
was:
{code:java}
from pyspark.sql.functions import flatten, struct, transform
df = self.spark.sql("SELECT array(1, 2, 3) as numbers, array('a', 'b', 'c') as
letters")
actual = df.select(
flatten(
transform(
"numbers",
lambda number: transform(
"letters", lambda letter: struct(number.alias("n"),
letter.alias("l"))
),
)
)
).first()[0]
expected = [
(1, "a"),
(1, "b"),
(1, "c"),
(2, "a"),
(2, "b"),
(2, "c"),
(3, "a"),
(3, "b"),
(3, "c"),
]
self.assertEquals(actual, expected){code}
{code:java}
Traceback (most recent call last):
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/tests/test_functions.py",
line 809, in test_nested_higher_order_function
self.assertEquals(actual, expected)
AssertionError: Lists differ: [{'n': 'a', 'l': 'a'}, {'n': 'b', 'l': 'b'[151
chars]'c'}] != [(1, 'a'), (1, 'b'), (1, 'c'), (2, 'a'), ([43 chars]'c')]
First differing element 0:
{'n': 'a', 'l': 'a'}
(1, 'a')
- [{'l': 'a', 'n': 'a'},
- {'l': 'b', 'n': 'b'},
- {'l': 'c', 'n': 'c'},
- {'l': 'a', 'n': 'a'},
- {'l': 'b', 'n': 'b'},
- {'l': 'c', 'n': 'c'},
- {'l': 'a', 'n': 'a'},
- {'l': 'b', 'n': 'b'},
- {'l': 'c', 'n': 'c'}]
+ [(1, 'a'),
+ (1, 'b'),
+ (1, 'c'),
+ (2, 'a'),
+ (2, 'b'),
+ (2, 'c'),
+ (3, 'a'),
+ (3, 'b'),
+ (3, 'c')]
{code}
> Fix `nth_value` functions output
> --------------------------------
>
> Key: SPARK-41904
> URL: https://issues.apache.org/jira/browse/SPARK-41904
> Project: Spark
> Issue Type: Sub-task
> Components: Connect
> Affects Versions: 3.4.0
> Reporter: Sandeep Singh
> Priority: Major
>
> {code:java}
> from pyspark.sql import Window
> from pyspark.sql.functions import nth_value
> df = self.spark.createDataFrame(
> [
> ("a", 0, None),
> ("a", 1, "x"),
> ("a", 2, "y"),
> ("a", 3, "z"),
> ("a", 4, None),
> ("b", 1, None),
> ("b", 2, None),
> ],
> schema=("key", "order", "value"),
> )
> w = Window.partitionBy("key").orderBy("order")
> rs = df.select(
> df.key,
> df.order,
> nth_value("value", 2).over(w),
> nth_value("value", 2, False).over(w),
> nth_value("value", 2, True).over(w),
> ).collect()
> expected = [
> ("a", 0, None, None, None),
> ("a", 1, "x", "x", None),
> ("a", 2, "x", "x", "y"),
> ("a", 3, "x", "x", "y"),
> ("a", 4, "x", "x", "y"),
> ("b", 1, None, None, None),
> ("b", 2, None, None, None),
> ]
> for r, ex in zip(sorted(rs), sorted(expected)):
> self.assertEqual(tuple(r), ex[: len(r)]){code}
> {code:java}
> Traceback (most recent call last):
> File
> "/Users/s.singh/personal/spark-oss/python/pyspark/sql/tests/test_functions.py",
> line 755, in test_nth_value
> self.assertEqual(tuple(r), ex[: len(r)])
> AssertionError: Tuples differ: ('a', 1, 'x', None) != ('a', 1, 'x', 'x')
> First differing element 3:
> None
> 'x'
> - ('a', 1, 'x', None)
> ? ^^^^
> + ('a', 1, 'x', 'x')
> ? ^^^
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
