HyukjinKwon commented on code in PR #59038:
URL: https://github.com/apache/spark/pull/59038#discussion_r4132795386
##########
python/pyspark/sql/tests/test_types.py:
##########
@@ -305,6 +306,42 @@ def test_infer_schema_upcast_boolean_to_string(self):
df = self.spark.createDataFrame([[True, 1], ["false", 1]],
schema=["a", "b"])
self.assertEqual([Row(a="true", b=1), Row(a="false", b=1)],
df.collect())
+ def test_infer_schema_row_length_mismatch(self):
+ # SPARK-59781: the schema is inferred from the first row. A later row
with more values used
+ # to be silently truncated when a field needed a converter (the string
column here), and a
+ # Row with an extra key came back with the values shifted (Row(a="3",
b=None)).
+ for data in [
+ [("a", 1), ("b", 2, 3)],
+ [Row(a="x", b=1), Row(c=3, a="y", b=2)],
+ [("a", 1), ("b",)],
+ ]:
+ with self.subTest(data=data):
+ rdd = self.sc.parallelize(data)
+ with self.assertRaisesRegex(PythonException,
"FIELD_STRUCT_LENGTH_MISMATCH"):
+ self.spark.createDataFrame(rdd).collect()
+
+ def test_create_dataframe_row_length_mismatch_without_verification(self):
+ # SPARK-59781: with verifySchema=False the type verifier is skipped
and a row longer than
Review Comment:
Actually, shouldn't we better just document this? `verifySchema` will
truncate rows too. To prevent this, enable `verifySchema` option.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
