Github user ueshin commented on a diff in the pull request:
https://github.com/apache/spark/pull/20678#discussion_r170792766
--- Diff: python/pyspark/sql/session.py ---
@@ -666,8 +666,28 @@ def createDataFrame(self, data, schema=None,
samplingRatio=None, verifySchema=Tr
try:
return self._create_from_pandas_with_arrow(data,
schema, timezone)
except Exception as e:
- warnings.warn("Arrow will not be used in
createDataFrame: %s" % str(e))
- # Fallback to create DataFrame without arrow if raise
some exception
+ from pyspark.util import _exception_message
+
+ if
self.conf.get("spark.sql.execution.arrow.fallback.enabled", "false") \
--- End diff --
ditto.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
