MaxGekk commented on code in PR #44939:
URL: https://github.com/apache/spark/pull/44939#discussion_r1471368855
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala:
##########
@@ -3248,6 +3249,58 @@ abstract class CSVSuite
}
}
}
+
+ test("SPARK-46890: CSV fails on a column with default and without enforcing
schema") {
+ withTable("CarsTable") {
+ spark.sql(
+ s"""
+ |CREATE TABLE CarsTable(
+ | year INT,
+ | make STRING,
+ | model STRING,
+ | comment STRING DEFAULT '',
+ | blank STRING DEFAULT '')
+ |USING csv
+ |OPTIONS (
+ | header "true",
+ | inferSchema "false",
+ | enforceSchema "false",
+ | path "${testFile(carsFile)}"
+ |)
+ """.stripMargin)
+ checkAnswer(
+ sql("SELECT comment FROM CarsTable"),
+ Seq(
+ Row("No comment"),
+ Row("Go get one now they are going fast"),
+ Row("")
+ ))
+ }
+ withTable("Products") {
+ spark.sql(
+ s"""
+ |CREATE TABLE IF NOT EXISTS Products (
+ | product_id INT,
+ | name STRING,
+ | price FLOAT default 0.0,
+ | quantity INT default 0
+ |)
+ |USING CSV
+ |OPTIONS (
+ | header 'true',
+ | inferSchema 'false',
+ | enforceSchema 'false',
+ | path "${testFile(productsFile)}"
Review Comment:
If this data is used only once, please, create a temporary file as in other
tests in the suite.
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala:
##########
@@ -3248,6 +3249,58 @@ abstract class CSVSuite
}
}
}
+
+ test("SPARK-46890: CSV fails on a column with default and without enforcing
schema") {
+ withTable("CarsTable") {
+ spark.sql(
+ s"""
+ |CREATE TABLE CarsTable(
+ | year INT,
+ | make STRING,
+ | model STRING,
+ | comment STRING DEFAULT '',
+ | blank STRING DEFAULT '')
+ |USING csv
+ |OPTIONS (
+ | header "true",
+ | inferSchema "false",
+ | enforceSchema "false",
+ | path "${testFile(carsFile)}"
+ |)
+ """.stripMargin)
+ checkAnswer(
Review Comment:
Why do you need this check? The next one reproduces the issue, doesn't it?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
