Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/20678#discussion_r170902707
--- Diff: python/pyspark/sql/tests.py ---
@@ -3493,19 +3514,30 @@ def create_pandas_data_frame(self):
data_dict["4_float_t"] = np.float32(data_dict["4_float_t"])
return pd.DataFrame(data=data_dict)
- def test_unsupported_datatype(self):
- schema = StructType([StructField("map", MapType(StringType(),
IntegerType()), True)])
- df = self.spark.createDataFrame([(None,)], schema=schema)
- with QuietTest(self.sc):
- with self.assertRaisesRegexp(Exception, 'Unsupported type'):
- df.toPandas()
+ def test_toPandas_fallback_enabled(self):
+ import pandas as pd
- df = self.spark.createDataFrame([(None,)], schema="a binary")
- with QuietTest(self.sc):
- with self.assertRaisesRegexp(
- Exception,
- 'Unsupported type.*\nNote: toPandas attempted Arrow
optimization because'):
- df.toPandas()
+ with self.sql_conf("spark.sql.execution.arrow.fallback.enabled",
True):
+ schema = StructType([StructField("map", MapType(StringType(),
IntegerType()), True)])
+ df = self.spark.createDataFrame([({u'a': 1},)], schema=schema)
+ with QuietTest(self.sc):
+ with warnings.catch_warnings(record=True) as warns:
+ pdf = df.toPandas()
+ # Catch and check the last UserWarning.
+ user_warns = [
+ warn.message for warn in warns if
isinstance(warn.message, UserWarning)]
+ self.assertTrue(len(user_warns) > 0)
+ self.assertTrue(
+ "Attempts non-optimization" in
_exception_message(user_warns[-1]))
+ self.assertPandasEqual(pdf, pd.DataFrame({u'map':
[{u'a': 1}]}))
+
+ def test_toPandas_fallback_disabled(self):
+ with self.sql_conf("spark.sql.execution.arrow.fallback.enabled",
False):
--- End diff --
Yea, I was thinking that too. I took a quick look for the rest of tests and
seems we are fine with a single pair for now. Will fix it as so in place in the
future if you are okay with that too.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
