This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new aad0f0010c8a [SPARK-53448][PYTHON] Conversion of a pyspark DataFrame
with a Variant column to pandas fails with an error
aad0f0010c8a is described below
commit aad0f0010c8a4f2091286db668781bf53ddb5e51
Author: VINDHYA G BHAT <[email protected]>
AuthorDate: Tue Nov 11 09:03:07 2025 -0800
[SPARK-53448][PYTHON] Conversion of a pyspark DataFrame with a Variant
column to pandas fails with an error
### What changes were proposed in this pull request?
When a dataframe is converted to Pandas, VariantVal is sent to the
conversion function which is not checked resulting to error. This PR aims to
fix it by returning the VariantVal value itself.
### Why are the changes needed?
df_variant = df.withColumn("variant_column", expr("parse_json(json_data)"))
pdf = df_variant.toPandas()
Above code returns error if there is no check for VariantVal value in
convert varianttype method.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Tested locally by doing the changes and running the code
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #52260 from VindhyaG/SPARK-53448.
Lead-authored-by: VINDHYA G BHAT <[email protected]>
Co-authored-by: Vindhya G <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit 9ff0fba06e758d2509dd8eb7a38b01cc8720d43d)
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/sql/pandas/types.py | 6 ++++--
python/pyspark/sql/tests/test_types.py | 17 +++++++++++++++++
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/python/pyspark/sql/pandas/types.py
b/python/pyspark/sql/pandas/types.py
index d8a45daa77e8..9583c8ac7288 100644
--- a/python/pyspark/sql/pandas/types.py
+++ b/python/pyspark/sql/pandas/types.py
@@ -23,6 +23,8 @@ import datetime
import itertools
from typing import Any, Callable, Iterable, List, Optional, Union,
TYPE_CHECKING
+from pyspark.errors import PySparkTypeError, UnsupportedOperationException,
PySparkValueError
+from pyspark.loose_version import LooseVersion
from pyspark.sql.types import (
cast,
BooleanType,
@@ -56,8 +58,6 @@ from pyspark.sql.types import (
Geography,
_create_row,
)
-from pyspark.errors import PySparkTypeError, UnsupportedOperationException,
PySparkValueError
-from pyspark.loose_version import LooseVersion
if TYPE_CHECKING:
import pandas as pd
@@ -1157,6 +1157,8 @@ def _create_converter_to_pandas(
elif isinstance(dt, VariantType):
def convert_variant(value: Any) -> Any:
+ if isinstance(value, VariantVal):
+ return value
if (
isinstance(value, dict)
and all(key in value for key in ["value", "metadata"])
diff --git a/python/pyspark/sql/tests/test_types.py
b/python/pyspark/sql/tests/test_types.py
index 4ff2ab3e5cd7..0a5219202a3a 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -2452,6 +2452,23 @@ class TypesTestsMixin:
with self.assertRaises(PySparkValueError, msg="Rows cannot be of type
VariantVal"):
self.spark.createDataFrame([VariantVal.parseJson("2")], "v
variant")
+ def test_variant_to_pandas(self):
+ import pandas as pd
+ import json
+
+ expected_values = [
+ ("str", '"%s"' % ("0123456789" * 10), "0123456789" * 10),
+ ("short_str", '"abc"', "abc"),
+ ]
+ json_str = "{%s}" % ",".join(['"%s": %s' % (t[0], t[1]) for t in
expected_values])
+ df = self.spark.createDataFrame([({"json": json_str})])
+ df_variant = df.select(F.parse_json(df.json).alias("v"))
+ pandas = df_variant.toPandas()
+ test_record = json.loads(pandas["v"].iloc[0].toJson())
+ self.assertIsInstance(pandas, pd.DataFrame)
+ self.assertEqual(expected_values[0][2], test_record["str"])
+ self.assertEqual(expected_values[1][2], test_record["short_str"])
+
def test_geospatial_encoding(self):
df = self.spark.createDataFrame(
[
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
