This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 212d0a7aef2f [SPARK-54330][PYTHON][FOLLOW-UP] Fetch
`spark.sql.execution.pandas.inferPandasDictAsMap` with other configs
212d0a7aef2f is described below
commit 212d0a7aef2f8c7937257b30b7b8c5daa0f71cd7
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Thu Dec 4 10:28:23 2025 -0800
[SPARK-54330][PYTHON][FOLLOW-UP] Fetch
`spark.sql.execution.pandas.inferPandasDictAsMap` with other configs
### What changes were proposed in this pull request?
Fetch `spark.sql.execution.pandas.inferPandasDictAsMap` with other configs
### Why are the changes needed?
to save 1 py4j call
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
ci
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #53323 from zhengruifeng/conv_inferPandasDictAsMap.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
python/pyspark/sql/pandas/conversion.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/python/pyspark/sql/pandas/conversion.py
b/python/pyspark/sql/pandas/conversion.py
index d34a9cdb53f0..991c143e5232 100644
--- a/python/pyspark/sql/pandas/conversion.py
+++ b/python/pyspark/sql/pandas/conversion.py
@@ -497,6 +497,7 @@ class SparkConversionMixin:
arrowPySparkFallbackEnabled,
arrowMaxRecordsPerBatch,
arrowSafeTypeConversion,
+ inferPandasDictAsMap,
) = self._jconf.getConfs(
[
"spark.sql.timestampType",
@@ -506,6 +507,7 @@ class SparkConversionMixin:
"spark.sql.execution.arrow.pyspark.fallback.enabled",
"spark.sql.execution.arrow.maxRecordsPerBatch",
"spark.sql.execution.pandas.convertToArrowArraySafely",
+ "spark.sql.execution.pandas.inferPandasDictAsMap",
]
)
@@ -514,6 +516,7 @@ class SparkConversionMixin:
timezone = sessionLocalTimeZone
arrow_batch_size = int(arrowMaxRecordsPerBatch)
selfcheck = arrowSafeTypeConversion == "true"
+ infer_pandas_dict_as_map = inferPandasDictAsMap == "true"
if type(data).__name__ == "Table":
# `data` is a PyArrow Table
@@ -557,6 +560,7 @@ class SparkConversionMixin:
prefers_large_var_types,
arrow_batch_size,
selfcheck,
+ infer_pandas_dict_as_map,
)
except Exception as e:
if arrowPySparkFallbackEnabled == "true":
@@ -788,6 +792,7 @@ class SparkConversionMixin:
prefers_large_var_types: bool,
arrow_batch_size: int,
safecheck: bool,
+ infer_pandas_dict_as_map: bool,
) -> "DataFrame":
"""
Create a DataFrame from a given pandas.DataFrame by slicing it into
partitions, converting
@@ -820,10 +825,6 @@ class SparkConversionMixin:
)
import pyarrow as pa
- infer_pandas_dict_as_map = (
-
str(self.conf.get("spark.sql.execution.pandas.inferPandasDictAsMap")).lower()
== "true"
- )
-
# Create the Spark schema from list of names passed in with Arrow types
if isinstance(schema, (list, tuple)):
arrow_schema = pa.Schema.from_pandas(pdf, preserve_index=False)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
