Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/19459#discussion_r150229857
--- Diff: python/pyspark/sql/tests.py ---
@@ -3180,6 +3185,58 @@ def test_filtered_frame(self):
self.assertEqual(pdf.columns[0], "i")
self.assertTrue(pdf.empty)
+ def test_createDataFrame_toggle(self):
+ pdf = self.create_pandas_data_frame()
+ self.spark.conf.set("spark.sql.execution.arrow.enabled", "false")
+ try:
+ df_no_arrow = self.spark.createDataFrame(pdf)
+ finally:
+ self.spark.conf.set("spark.sql.execution.arrow.enabled",
"true")
+ df_arrow = self.spark.createDataFrame(pdf)
+ self.assertEquals(df_no_arrow.collect(), df_arrow.collect())
+
+ def test_createDataFrame_with_schema(self):
+ pdf = self.create_pandas_data_frame()
+ df = self.spark.createDataFrame(pdf, schema=self.schema)
+ self.assertEquals(self.schema, df.schema)
+ pdf_arrow = df.toPandas()
+ self.assertFramesEqual(pdf_arrow, pdf)
+
+ def test_createDataFrame_with_incorrect_schema(self):
+ pdf = self.create_pandas_data_frame()
+ wrong_schema = StructType([field for field in
reversed(self.schema)])
--- End diff --
Not a big deal at all: `StructType([field for field in
reversed(self.schema)])` -> `StructType(list(reversed(st)))`
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
