This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 50af717 Revert "[SPARK-37575][SQL] null values should be saved as
nothing rather than quoted empty Strings "" by default settings"
50af717 is described below
commit 50af71747be96b34f36b969e023237c7c5c0f31c
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Wed Dec 15 12:54:28 2021 +0900
Revert "[SPARK-37575][SQL] null values should be saved as nothing rather
than quoted empty Strings "" by default settings"
This reverts commit 62e4202b65d76b05f9f9a15819a631524c6e7985.
---
.../apache/spark/sql/catalyst/csv/UnivocityGenerator.scala | 2 ++
.../spark/sql/execution/datasources/csv/CSVSuite.scala | 13 +------------
2 files changed, 3 insertions(+), 12 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
index 8504877..2abf7bf 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
@@ -84,6 +84,8 @@ class UnivocityGenerator(
while (i < row.numFields) {
if (!row.isNullAt(i)) {
values(i) = valueConverters(i).apply(row, i)
+ } else {
+ values(i) = options.nullValue
}
i += 1
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index a472221..7efdf7c 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -804,17 +804,6 @@ abstract class CSVSuite
}
}
- test("SPARK-37575: null values should be saved as nothing rather than " +
- "quoted empty Strings \"\" with default settings") {
- withTempPath { path =>
- Seq(("Tesla", null: String, ""))
- .toDF("make", "comment", "blank")
- .write
- .csv(path.getCanonicalPath)
- checkAnswer(spark.read.text(path.getCanonicalPath), Row("Tesla,,\"\""))
- }
- }
-
test("save csv with compression codec option") {
withTempDir { dir =>
val csvDir = new File(dir, "csv").getCanonicalPath
@@ -1585,7 +1574,7 @@ abstract class CSVSuite
(1, "John Doe"),
(2, "-"),
(3, "-"),
- (4, null)
+ (4, "-")
).toDF("id", "name")
checkAnswer(computed, expected)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
