MaxGekk commented on code in PR #47906:
URL: https://github.com/apache/spark/pull/47906#discussion_r1739606803
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala:
##########
@@ -323,6 +323,41 @@ class UnivocityParserSuite extends SparkFunSuite with
SQLHelper {
parameters = Map("fieldName" -> "`i`", "fields" -> ""))
}
+ test("Bad records test in permissive mode") {
+ def checkBadRecord(
+ input: String = "1,a",
+ dataSchema: StructType = StructType.fromDDL("i INTEGER, s STRING, d
DOUBLE"),
+ requiredSchema: StructType = StructType.fromDDL("i INTEGER, s STRING"),
+ options: Map[String, String] = Map("mode" -> "PERMISSIVE")):
BadRecordException = {
+ val csvOptions = new CSVOptions(options, false, "UTC")
+ val parser = new UnivocityParser(dataSchema, requiredSchema, csvOptions,
Seq())
+ intercept[BadRecordException] {
+ parser.parse(input)
+ }
+ }
+
+ // Bad record exception caused by conversion error
+ checkBadRecord(input = "1.5,a,10.3")
+
+ // Bad record exception caused by insufficient number of columns
+ checkBadRecord(input = "2")
+ }
+
+ test("Array index out of bounds when parsing CSV with more columns than
expected") {
+ val input = "1,string,3.14,5,7"
+ val dataSchema: StructType = StructType.fromDDL("i INTEGER, a STRING")
+ val requiredSchema: StructType = StructType.fromDDL("i INTEGER, a STRING")
+ val options = new CSVOptions(Map("maxColumns" -> "2"), false, "UTC")
+ val filters = Seq()
+ val parser = new UnivocityParser(dataSchema, requiredSchema, options,
filters)
+ checkError(
+ exception = intercept[SparkRuntimeException] {
+ parser.parse(input)
+ },
+ errorClass = "MALFORMED_CSV_RECORD",
+ parameters = Map("badRecord" -> "1,string,3.14,5,7"))
Review Comment:
The error condition name seems misleading to me. The CSV record is not
malformed, but we actually hit to some limits of implementation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
