srielau commented on code in PR #58255:
URL: https://github.com/apache/spark/pull/58255#discussion_r3865248737
##########
sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala:
##########
@@ -2157,6 +2157,43 @@ class BasicCharVarcharTestSuite extends
SharedSparkSession {
}
}
}
+
+ test("SPARK-59001: empty CHAR/VARCHAR partition values become null like
STRING") {
+ withSQLConf(SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key -> "true") {
+ // CHAR(n>0) pads '' to spaces; CHAR(0) is the empty CHAR that
empty2null should treat
+ // the same as VARCHAR/STRING.
+ Seq("CHAR(0)", "VARCHAR(5)").foreach { typ =>
+ withTempPath { path =>
+ sql(s"SELECT 0 AS id, CAST('' AS $typ) AS p UNION ALL SELECT 1,
CAST(NULL AS $typ)")
+
.write.mode("overwrite").partitionBy("p").parquet(path.getCanonicalPath)
+ val df = spark.read.parquet(path.getCanonicalPath)
+ checkAnswer(df.where("p IS NULL").select("id"), Seq(Row(0), Row(1)))
+ val dirs = path.listFiles().filterNot(
+ f => f.getName.startsWith(".") || f.getName.startsWith("_"))
+ assert(dirs.length === 1, dirs.map(_.getName).mkString(","))
+ }
+ }
+ }
+ }
+
+ test("SPARK-59001: text datasource accepts CHAR/VARCHAR as a string family
type") {
Review Comment:
Addressed in 5270f4b20ed. The regression test now runs with both
`USE_V1_SOURCE_LIST = "text"` and `USE_V1_SOURCE_LIST = ""`, so it exercises
`TextFileFormat` and `TextTable.supportsDataType`. The targeted SPARK-59001
text test passes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
