Github user HyukjinKwon commented on a diff in the pull request:

    https://github.com/apache/spark/pull/20678#discussion_r172540555
  
    --- Diff: python/pyspark/sql/tests.py ---
    @@ -3493,19 +3519,30 @@ def create_pandas_data_frame(self):
             data_dict["4_float_t"] = np.float32(data_dict["4_float_t"])
             return pd.DataFrame(data=data_dict)
     
    -    def test_unsupported_datatype(self):
    -        schema = StructType([StructField("map", MapType(StringType(), 
IntegerType()), True)])
    -        df = self.spark.createDataFrame([(None,)], schema=schema)
    -        with QuietTest(self.sc):
    -            with self.assertRaisesRegexp(Exception, 'Unsupported type'):
    -                df.toPandas()
    +    def test_toPandas_fallback_enabled(self):
    +        import pandas as pd
     
    -        df = self.spark.createDataFrame([(None,)], schema="a binary")
    -        with QuietTest(self.sc):
    -            with self.assertRaisesRegexp(
    -                    Exception,
    -                    'Unsupported type.*\nNote: toPandas attempted Arrow 
optimization because'):
    -                df.toPandas()
    +        with self.sql_conf({"spark.sql.execution.arrow.fallback.enabled": 
True}):
    +            schema = StructType([StructField("map", MapType(StringType(), 
IntegerType()), True)])
    +            df = self.spark.createDataFrame([({u'a': 1},)], schema=schema)
    +            with QuietTest(self.sc):
    +                with warnings.catch_warnings(record=True) as warns:
    +                    pdf = df.toPandas()
    +                    # Catch and check the last UserWarning.
    +                    user_warns = [
    +                        warn.message for warn in warns if 
isinstance(warn.message, UserWarning)]
    +                    self.assertTrue(len(user_warns) > 0)
    +                    self.assertTrue(
    +                        "Attempts non-optimization" in 
_exception_message(user_warns[-1]))
    +                    self.assertPandasEqual(pdf, pd.DataFrame({u'map': 
[{u'a': 1}]}))
    +
    +    def test_toPandas_fallback_disabled(self):
    +        with self.sql_conf({"spark.sql.execution.arrow.fallback.enabled": 
False}):
    --- End diff --
    
    Hm .. yup. I don't feel strongly. Will remove it out.


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

Reply via email to