[
https://issues.apache.org/jira/browse/SPARK-41837?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sandeep Singh updated SPARK-41837:
----------------------------------
Description:
{code:java}
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
line 1594, in pyspark.sql.connect.functions.to_json
Failed example:
df = spark.createDataFrame(data, ("key", "value"))
Exception raised:
Traceback (most recent call last):
File
"/usr/local/Cellar/[email protected]/3.10.8/Frameworks/Python.framework/Versions/3.10/lib/python3.10/doctest.py",
line 1350, in __run
exec(compile(example.source, filename, "single",
File "<doctest pyspark.sql.connect.functions.to_json[3]>", line 1, in
<module>
df = spark.createDataFrame(data, ("key", "value"))
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/session.py", line
252, in createDataFrame
table = pa.Table.from_pandas(pdf)
File "pyarrow/table.pxi", line 3475, in pyarrow.lib.Table.from_pandas
File "/usr/local/lib/python3.10/site-packages/pyarrow/pandas_compat.py",
line 611, in dataframe_to_arrays
arrays = [convert_column(c, f)
File "/usr/local/lib/python3.10/site-packages/pyarrow/pandas_compat.py",
line 611, in <listcomp>
arrays = [convert_column(c, f)
File "/usr/local/lib/python3.10/site-packages/pyarrow/pandas_compat.py",
line 598, in convert_column
raise e
File "/usr/local/lib/python3.10/site-packages/pyarrow/pandas_compat.py",
line 592, in convert_column
result = pa.array(col, type=type_, from_pandas=True, safe=safe)
File "pyarrow/array.pxi", line 316, in pyarrow.lib.array
File "pyarrow/array.pxi", line 83, in pyarrow.lib._ndarray_to_array
File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: ("Could not convert 'Alice' with type str: tried
to convert to int64", 'Conversion failed for column 1 with type object'){code}
was:
{code:java}
**********************************************************************
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
line 1117, in pyspark.sql.connect.functions.array
Failed example:
df.select(array('age', 'age').alias("arr")).collect()
Expected:
[Row(arr=[2, 2]), Row(arr=[5, 5])]
Got:
[Row(arr=array([2, 2])), Row(arr=array([5, 5]))]
**********************************************************************
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
line 1119, in pyspark.sql.connect.functions.array
Failed example:
df.select(array([df.age, df.age]).alias("arr")).collect()
Expected:
[Row(arr=[2, 2]), Row(arr=[5, 5])]
Got:
[Row(arr=array([2, 2])), Row(arr=array([5, 5]))]
**********************************************************************
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
line 1124, in pyspark.sql.connect.functions.array_distinct
Failed example:
df.select(array_distinct(df.data)).collect()
Expected:
[Row(array_distinct(data)=[1, 2, 3]), Row(array_distinct(data)=[4, 5])]
Got:
[Row(array_distinct(data)=array([1, 2, 3])),
Row(array_distinct(data)=array([4, 5]))]
**********************************************************************
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
line 1135, in pyspark.sql.connect.functions.array_except
Failed example:
df.select(array_except(df.c1, df.c2)).collect()
Expected:
[Row(array_except(c1, c2)=['b'])]
Got:
[Row(array_except(c1, c2)=array(['b'], dtype=object))]
**********************************************************************
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
line 1142, in pyspark.sql.connect.functions.array_intersect
Failed example:
df.select(array_intersect(df.c1, df.c2)).collect()
Expected:
[Row(array_intersect(c1, c2)=['a', 'c'])]
Got:
[Row(array_intersect(c1, c2)=array(['a', 'c'], dtype=object))]
**********************************************************************
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
line 1180, in pyspark.sql.connect.functions.array_remove
Failed example:
df.select(array_remove(df.data, 1)).collect()
Expected:
[Row(array_remove(data, 1)=[2, 3]), Row(array_remove(data, 1)=[])]
Got:
[Row(array_remove(data, 1)=array([2, 3])), Row(array_remove(data,
1)=array([], dtype=int64))]
**********************************************************************
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
line 1187, in pyspark.sql.connect.functions.array_repeat
Failed example:
df.select(array_repeat(df.data, 3).alias('r')).collect()
Expected:
[Row(r=['ab', 'ab', 'ab'])]
Got:
[Row(r=array(['ab', 'ab', 'ab'], dtype=object))]
**********************************************************************
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
line 1204, in pyspark.sql.connect.functions.array_sort
Failed example:
df.select(array_sort(df.data).alias('r')).collect()
Expected:
[Row(r=[1, 2, 3, None]), Row(r=[1]), Row(r=[])]
Got:
[Row(r=array([ 1., 2., 3., nan])), Row(r=array([1])), Row(r=array([],
dtype=int64))]
**********************************************************************
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
line 1207, in pyspark.sql.connect.functions.array_sort
Failed example:
df.select(array_sort(
"data",
lambda x, y: when(x.isNull() | y.isNull(), lit(0)).otherwise(length(y)
- length(x))
).alias("r")).collect()
Expected:
[Row(r=['foobar', 'foo', None, 'bar']), Row(r=['foo']), Row(r=[])]
Got:
[Row(r=array(['foobar', 'foo', None, 'bar'], dtype=object)),
Row(r=array(['foo'], dtype=object)), Row(r=array([], dtype=object))]
**********************************************************************
File
"/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
line 1209, in pyspark.sql.connect.functions.array_union
Failed example:
df.select(array_union(df.c1, df.c2)).collect()
Expected:
[Row(array_union(c1, c2)=['b', 'a', 'c', 'd', 'f'])]
Got:
[Row(array_union(c1, c2)=array(['b', 'a', 'c', 'd', 'f'],
dtype=object))]{code}
> DataFrame.createDataFrame datatype conversion error
> ---------------------------------------------------
>
> Key: SPARK-41837
> URL: https://issues.apache.org/jira/browse/SPARK-41837
> Project: Spark
> Issue Type: Sub-task
> Components: Connect
> Affects Versions: 3.4.0
> Reporter: Sandeep Singh
> Priority: Major
>
> {code:java}
> File
> "/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/functions.py",
> line 1594, in pyspark.sql.connect.functions.to_json
> Failed example:
> df = spark.createDataFrame(data, ("key", "value"))
> Exception raised:
> Traceback (most recent call last):
> File
> "/usr/local/Cellar/[email protected]/3.10.8/Frameworks/Python.framework/Versions/3.10/lib/python3.10/doctest.py",
> line 1350, in __run
> exec(compile(example.source, filename, "single",
> File "<doctest pyspark.sql.connect.functions.to_json[3]>", line 1, in
> <module>
> df = spark.createDataFrame(data, ("key", "value"))
> File
> "/Users/s.singh/personal/spark-oss/python/pyspark/sql/connect/session.py",
> line 252, in createDataFrame
> table = pa.Table.from_pandas(pdf)
> File "pyarrow/table.pxi", line 3475, in pyarrow.lib.Table.from_pandas
> File
> "/usr/local/lib/python3.10/site-packages/pyarrow/pandas_compat.py", line 611,
> in dataframe_to_arrays
> arrays = [convert_column(c, f)
> File
> "/usr/local/lib/python3.10/site-packages/pyarrow/pandas_compat.py", line 611,
> in <listcomp>
> arrays = [convert_column(c, f)
> File
> "/usr/local/lib/python3.10/site-packages/pyarrow/pandas_compat.py", line 598,
> in convert_column
> raise e
> File
> "/usr/local/lib/python3.10/site-packages/pyarrow/pandas_compat.py", line 592,
> in convert_column
> result = pa.array(col, type=type_, from_pandas=True, safe=safe)
> File "pyarrow/array.pxi", line 316, in pyarrow.lib.array
> File "pyarrow/array.pxi", line 83, in pyarrow.lib._ndarray_to_array
> File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status
> pyarrow.lib.ArrowInvalid: ("Could not convert 'Alice' with type str:
> tried to convert to int64", 'Conversion failed for column 1 with type
> object'){code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
