This is an automated email from the ASF dual-hosted git repository.
garydgregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
The following commit(s) were added to refs/heads/master by this push:
new f9e7d792 Keep an escaped value that equals the null string (#626).
f9e7d792 is described below
commit f9e7d792dc6cb8a46f3e164e5cc50e38dc65264b
Author: Gary Gregory <[email protected]>
AuthorDate: Sat Aug 29 08:04:18 2026 -0400
Keep an escaped value that equals the null string (#626).
PR plus fixes to the PR to pass the build.
---
src/changes/changes.xml | 1 +
src/main/java/org/apache/commons/csv/CSVParser.java | 5 ++++-
src/main/java/org/apache/commons/csv/Lexer.java | 3 +++
src/main/java/org/apache/commons/csv/Token.java | 4 ++++
.../java/org/apache/commons/csv/CSVParserTest.java | 18 ++++++++++++++++++
.../java/org/apache/commons/csv/CSVPrinterTest.java | 20 ++++++++++++++++----
6 files changed, 46 insertions(+), 5 deletions(-)
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 463ad12e..f3189baa 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -76,6 +76,7 @@
<action type="fix" dev="ggregory" due-to="Gary Gregory, Naveed Khan">Fix
quadratic CSVFormat.printWithEscapes(Reader) delimiter look-ahead
(#631).</action>
<action type="fix" dev="ggregory" due-to="Gary Gregory">General Javadoc
improvements.</action>
<action type="fix" dev="ggregory" due-to="Gary Gregory, Naveed
Khan">Validate CSVFormat invariants when deserializing (#632).</action>
+ <action type="fix" dev="ggregory" due-to="Gary Gregory, saleem
malik">Keep an escaped value that equals the null string (#626).</action>
<!-- ADD -->
<action type="add" dev="ggregory" due-to="Gary Gregory, Indy, Sylvia van
Os" issue="CSV-307">Add an "Android Compatibility" section to the web
site.</action>
<action type="add" dev="ggregory" due-to="Ruiqi Dong, Gary Gregory"
issue="CSV-325">Add CSVParser.Builder.setByteOffset(long) (#604).</action>
diff --git a/src/main/java/org/apache/commons/csv/CSVParser.java
b/src/main/java/org/apache/commons/csv/CSVParser.java
index 6d2a3b99..eb4c0d32 100644
--- a/src/main/java/org/apache/commons/csv/CSVParser.java
+++ b/src/main/java/org/apache/commons/csv/CSVParser.java
@@ -819,8 +819,11 @@ public final class CSVParser implements
Iterable<CSVRecord>, Closeable {
final String nullString = format.getNullString();
final boolean strictQuoteMode = isStrictQuoteMode();
if (input.equals(nullString)) {
+ // A token that needed an escape translation cannot be the null
marker: printing null emits the null
+ // string without escaping it (it may be quoted, but never
escaped), so an escaped "\N" for nullString
+ // "\N" can only have come from a field whose value really is "\N".
// nullString = NULL(String), distinguish between "NULL" and NULL
in ALL_NON_NULL or NON_NUMERIC quote mode
- return strictQuoteMode && isQuoted ? input : null;
+ return reusableToken.isEscaped || strictQuoteMode && isQuoted ?
input : null;
}
// don't set nullString, distinguish between "" and ,, (absent values)
in All_NON_NULL or NON_NUMERIC quote mode
return strictQuoteMode && nullString == null && input.isEmpty() &&
!isQuoted ? null : input;
diff --git a/src/main/java/org/apache/commons/csv/Lexer.java
b/src/main/java/org/apache/commons/csv/Lexer.java
index c78d1c7b..12f1aa06 100644
--- a/src/main/java/org/apache/commons/csv/Lexer.java
+++ b/src/main/java/org/apache/commons/csv/Lexer.java
@@ -76,12 +76,15 @@ final class Lexer implements Closeable {
private void appendNextEscapedCharacterToToken(final Token token) throws
IOException {
if (isEscapeDelimiter()) {
token.content.append(delimiter);
+ token.isEscaped = true;
} else {
final int unescaped = readEscape();
if (unescaped == EOF) { // unexpected char after escape
+ // The escape character is kept verbatim, so nothing was
translated.
token.content.append((char) escape).append((char)
reader.getLastChar());
} else {
token.content.append((char) unescaped);
+ token.isEscaped = true;
}
}
}
diff --git a/src/main/java/org/apache/commons/csv/Token.java
b/src/main/java/org/apache/commons/csv/Token.java
index 42c4e3bd..bde790a6 100644
--- a/src/main/java/org/apache/commons/csv/Token.java
+++ b/src/main/java/org/apache/commons/csv/Token.java
@@ -61,11 +61,15 @@ final class Token {
boolean isQuoted;
+ /** True when an escape sequence in the input was translated while
building {@link #content}. */
+ boolean isEscaped;
+
void reset() {
content.setLength(0);
type = INVALID;
isReady = false;
isQuoted = false;
+ isEscaped = false;
}
/**
diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java
b/src/test/java/org/apache/commons/csv/CSVParserTest.java
index 5300a63a..50a6ae03 100644
--- a/src/test/java/org/apache/commons/csv/CSVParserTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java
@@ -662,6 +662,24 @@ class CSVParserTest {
}
}
+ @ParameterizedTest
+ @EnumSource(value = CSVFormat.Predefined.class, names = { "MySQL",
"PostgreSQLCsv", "PostgreSQLText", "Oracle" })
+ void testEscapedNullStringIsAValue(final CSVFormat.Predefined predefined)
throws Exception {
+ // "\N" is the null string for MySQL, PostgreSQL Text and Oracle;
PostgreSQL CSV uses an empty null
+ // string. In every case a field whose value equals "\N" must round
trip as that value, not as null.
+ final String valueEqualToNullString = "\\N";
+ final CSVFormat format = predefined.getFormat();
+ final StringWriter writer = new StringWriter();
+ try (CSVPrinter printer = new CSVPrinter(writer, format)) {
+ printer.printRecord(valueEqualToNullString, null);
+ }
+ try (CSVParser parser = CSVParser.parse(writer.toString(), format)) {
+ final CSVRecord record = parser.nextRecord();
+ assertEquals(valueEqualToNullString, record.get(0));
+ assertNull(record.get(1));
+ }
+ }
+
@Test
void testExcelFormat1() throws IOException {
final String code = "value1,value2,value3,value4\r\na,b,c,d\r\n
x,,,\r\n\r\n\"\"\"hello\"\"\",\" \"\"world\"\"\",\"abc\ndef\",\r\n";
diff --git a/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
b/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
index 1c25821e..f1f910bb 100644
--- a/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
@@ -158,13 +158,18 @@ class CSVPrinterTest {
}
/**
- * Converts an input CSV array into expected output values, including
NULLs. NULL strings are converted to null values because the parser will convert
- * these strings to null.
+ * Converts an input CSV array into expected output values, including
NULLs. A value equal to the null string is expected back as null only when the
+ * printer writes it verbatim (neither escaped nor quoted); once it is
escaped or quoted the parser can tell it apart from a real null and reads it
back
+ * as the value it is.
*/
- private <T> T[] expectNulls(final T[] original, final CSVFormat csvFormat)
{
+ private <T> T[] expectNulls(final T[] original, final CSVFormat csvFormat)
throws IOException {
final T[] fixed = original.clone();
+ final String nullString = csvFormat.getNullString();
+ if (nullString == null || !printsVerbatim(csvFormat, nullString)) {
+ return fixed;
+ }
for (int i = 0; i < fixed.length; i++) {
- if (Objects.equals(csvFormat.getNullString(), fixed[i])) {
+ if (Objects.equals(nullString, fixed[i])) {
fixed[i] = null;
}
}
@@ -196,6 +201,13 @@ class CSVPrinterTest {
return sw.toString();
}
+ /** Tests whether the format prints the given value unchanged, in which
case the parser cannot tell it from the null string. */
+ private boolean printsVerbatim(final CSVFormat csvFormat, final String
value) throws IOException {
+ final StringBuilder sb = new StringBuilder();
+ csvFormat.print(value, sb, true);
+ return value.contentEquals(sb);
+ }
+
private CSVPrinter printWithHeaderComments(final StringWriter sw, final
Date now, final CSVFormat baseFormat) throws IOException {
// Use withHeaderComments first to test CSV-145
// @formatter:off
