Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/20894#discussion_r188921159
--- Diff: python/pyspark/sql/tests.py ---
@@ -3040,6 +3040,24 @@ def test_csv_sampling_ratio(self):
.csv(rdd, samplingRatio=0.5).schema
self.assertEquals(schema, StructType([StructField("_c0",
IntegerType(), True)]))
+ def test_checking_csv_header(self):
+ tmpPath = tempfile.mkdtemp()
+ shutil.rmtree(tmpPath)
+ try:
+ self.spark.createDataFrame([[1, 1000], [2000, 2]])\
+ .toDF('f1', 'f2').write.option("header",
"true").csv(tmpPath)
+ schema = StructType([
+ StructField('f2', IntegerType(), nullable=True),
+ StructField('f1', IntegerType(), nullable=True)])
+ df = self.spark.read.option('header', 'true').schema(schema)\
+ .csv(tmpPath, enforceSchema=False)
+ self.assertRaisesRegexp(
+ Exception,
+ "CSV file header does not contain the expected fields",
--- End diff --
the error message is a little confusing. We do contain the fields, but in
different order.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
