Sandeep Singh created SPARK-41902:
-------------------------------------
Summary: Fix String representation of maps created by
`map_from_arrays`
Key: SPARK-41902
URL: https://issues.apache.org/jira/browse/SPARK-41902
Project: Spark
Issue Type: Sub-task
Components: Connect
Affects Versions: 3.4.0
Reporter: Sandeep Singh
{code:java}
from pyspark.sql import functions
funs = [
(functions.acosh, "ACOSH"),
(functions.asinh, "ASINH"),
(functions.atanh, "ATANH"),
]
cols = ["a", functions.col("a")]
for f, alias in funs:
for c in cols:
self.assertIn(f"{alias}(a)", repr(f(c))){code}
{code:java}
Traceback (most recent call last):
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/tests/test_functions.py",
line 271, in test_inverse_trig_functions
self.assertIn(f"{alias}(a)", repr(f(c)))
AssertionError: 'ACOSH(a)' not found in
"Column<'acosh(ColumnReference(a))'>"{code}
{code:java}
from pyspark.sql.functions import col, lit, overlay
from itertools import chain
import re
actual = list(
chain.from_iterable(
[
re.findall("(overlay\\(.*\\))", str(x))
for x in [
overlay(col("foo"), col("bar"), 1),
overlay("x", "y", 3),
overlay(col("x"), col("y"), 1, 3),
overlay("x", "y", 2, 5),
overlay("x", "y", lit(11)),
overlay("x", "y", lit(2), lit(5)),
]
]
)
)
expected = [
"overlay(foo, bar, 1, -1)",
"overlay(x, y, 3, -1)",
"overlay(x, y, 1, 3)",
"overlay(x, y, 2, 5)",
"overlay(x, y, 11, -1)",
"overlay(x, y, 2, 5)",
]
self.assertListEqual(actual, expected)
df = self.spark.createDataFrame([("SPARK_SQL", "CORE", 7, 0)], ("x", "y",
"pos", "len"))
exp = [Row(ol="SPARK_CORESQL")]
self.assertTrue(
all(
[
df.select(overlay(df.x, df.y, 7, 0).alias("ol")).collect() == exp,
df.select(overlay(df.x, df.y, lit(7),
lit(0)).alias("ol")).collect() == exp,
df.select(overlay("x", "y", "pos", "len").alias("ol")).collect() ==
exp,
]
)
) {code}
{code:java}
Traceback (most recent call last):
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/tests/test_functions.py",
line 675, in test_overlay
self.assertListEqual(actual, expected)
AssertionError: Lists differ: ['overlay(ColumnReference(foo),
ColumnReference(bar[402 chars]5))'] != ['overlay(foo, bar, 1, -1)', 'overlay(x,
y, 3, -1)'[90 chars] 5)']
First differing element 0:
'overlay(ColumnReference(foo), ColumnReference(bar), Literal(1), Literal(-1))'
'overlay(foo, bar, 1, -1)'
- ['overlay(ColumnReference(foo), ColumnReference(bar), Literal(1),
Literal(-1))',
- 'overlay(ColumnReference(x), ColumnReference(y), Literal(3), Literal(-1))',
- 'overlay(ColumnReference(x), ColumnReference(y), Literal(1), Literal(3))',
- 'overlay(ColumnReference(x), ColumnReference(y), Literal(2), Literal(5))',
- 'overlay(ColumnReference(x), ColumnReference(y), Literal(11), Literal(-1))',
- 'overlay(ColumnReference(x), ColumnReference(y), Literal(2), Literal(5))']
+ ['overlay(foo, bar, 1, -1)',
+ 'overlay(x, y, 3, -1)',
+ 'overlay(x, y, 1, 3)',
+ 'overlay(x, y, 2, 5)',
+ 'overlay(x, y, 11, -1)',
+ 'overlay(x, y, 2, 5)']
{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
