This is an automated email from the ASF dual-hosted git repository.
garydgregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-csv.git
The following commit(s) were added to refs/heads/master by this push:
new fb71dcb6 Escape-mode printer and parser mishandles value ending in
multi-char delimiter prefix (f001).
fb71dcb6 is described below
commit fb71dcb6a1914664ce1bbd037199baf678b01f8c
Author: Gary Gregory <[email protected]>
AuthorDate: Mon Sep 7 07:17:32 2026 -0400
Escape-mode printer and parser mishandles value ending in multi-char
delimiter prefix (f001).
---
src/changes/changes.xml | 1 +
.../java/org/apache/commons/csv/CSVFormat.java | 83 ++++++++++++++++------
src/main/java/org/apache/commons/csv/Lexer.java | 23 +++++-
.../java/org/apache/commons/csv/CSVParserTest.java | 6 +-
.../org/apache/commons/csv/CSVPrinterTest.java | 75 +++++++++++++++++++
.../java/org/apache/commons/csv/LexerTest.java | 3 +-
6 files changed, 164 insertions(+), 27 deletions(-)
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index f3189baa..c7d17042 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -77,6 +77,7 @@
<action type="fix" dev="ggregory" due-to="Gary Gregory">General Javadoc
improvements.</action>
<action type="fix" dev="ggregory" due-to="Gary Gregory, Naveed
Khan">Validate CSVFormat invariants when deserializing (#632).</action>
<action type="fix" dev="ggregory" due-to="Gary Gregory, saleem
malik">Keep an escaped value that equals the null string (#626).</action>
+ <action type="fix" dev="ggregory" due-to="Gary Gregory">Escape-mode
printer and parser mishandles value ending in multi-char delimiter prefix
(f001).</action>
<!-- ADD -->
<action type="add" dev="ggregory" due-to="Gary Gregory, Indy, Sylvia van
Os" issue="CSV-307">Add an "Android Compatibility" section to the web
site.</action>
<action type="add" dev="ggregory" due-to="Ruiqi Dong, Gary Gregory"
issue="CSV-325">Add CSVParser.Builder.setByteOffset(long) (#604).</action>
diff --git a/src/main/java/org/apache/commons/csv/CSVFormat.java
b/src/main/java/org/apache/commons/csv/CSVFormat.java
index 891c424b..63698174 100644
--- a/src/main/java/org/apache/commons/csv/CSVFormat.java
+++ b/src/main/java/org/apache/commons/csv/CSVFormat.java
@@ -1481,6 +1481,20 @@ public final class CSVFormat implements Serializable {
return value == null || value.trim().isEmpty();
}
+ /**
+ * Tests whether every character of the delimiter stays itself when
individually escaped and read back. The letters used by control-character escape
+ * sequences ({@code r}, {@code n}, {@code t}, {@code b}, {@code f})
unescape to the control character, not to the letter, so a delimiter containing
one of
+ * them cannot have a straddling prefix escaped losslessly and keeps the
historical unescaped output.
+ */
+ private static boolean isDelimiterStraddleEscapable(final char[]
delimiter) {
+ for (final char ch : delimiter) {
+ if (ch == 'r' || ch == 'n' || ch == 't' || ch == 'b' || ch == 'f')
{
+ return false;
+ }
+ }
+ return true;
+ }
+
/**
* Returns true if the given character is a line break character.
*
@@ -1702,25 +1716,7 @@ public final class CSVFormat implements Serializable {
* delimiter yields {@code |||}, which the greedy lexer splits one
character early). Such a value must be encapsulated so the field boundary is
unambiguous.
*/
private boolean endsWithDelimiterPrefix(final CharSequence charSeq, final
char[] delimiter, final int delimiterLength) {
- if (delimiterLength < 2) {
- return false;
- }
- final int len = charSeq.length();
- for (int start = Math.max(0, len - delimiterLength + 1); start < len;
start++) {
- boolean match = true;
- for (int j = 0; j < delimiterLength; j++) {
- final int idx = start + j;
- final char c = idx < len ? charSeq.charAt(idx) : delimiter[idx
- len];
- if (c != delimiter[j]) {
- match = false;
- break;
- }
- }
- if (match) {
- return true;
- }
- }
- return false;
+ return indexOfDelimiterPrefix(charSeq, delimiter, delimiterLength) >=
0;
}
@Override
@@ -2092,6 +2088,33 @@ public final class CSVFormat implements Serializable {
return result;
}
+ /**
+ * Returns the index within {@code charSeq} where a straddling prefix of
the delimiter begins, or -1 if the value does not end with one. Every character
+ * from the returned index to the end of the value takes part in a
delimiter match that would start inside the value once the delimiter is
appended, so an
+ * escaping printer must escape each of them individually. See {@link
#endsWithDelimiterPrefix(CharSequence, char[], int)}.
+ */
+ private int indexOfDelimiterPrefix(final CharSequence charSeq, final
char[] delimiter, final int delimiterLength) {
+ if (delimiterLength < 2) {
+ return -1;
+ }
+ final int len = charSeq.length();
+ for (int start = Math.max(0, len - delimiterLength + 1); start < len;
start++) {
+ boolean match = true;
+ for (int j = 0; j < delimiterLength; j++) {
+ final int idx = start + j;
+ final char c = idx < len ? charSeq.charAt(idx) : delimiter[idx
- len];
+ if (c != delimiter[j]) {
+ match = false;
+ break;
+ }
+ }
+ if (match) {
+ return start;
+ }
+ }
+ return -1;
+ }
+
/**
* Tests whether comments are supported by this format.
*
@@ -2410,6 +2433,11 @@ public final class CSVFormat implements Serializable {
final char quote = quoteSet ? getQuoteCharacter().charValue() : 0;
final boolean commentMarkerSet = isCommentMarkerSet();
final char commentChar = commentMarkerSet ? commentMarker.charValue()
: 0; // Explicit unboxing is intentional
+ // A value ending in a straddling prefix of the delimiter must have
every character of that prefix escaped:
+ // appended after the bare prefix, the delimiter would match one
character early on read and shift the field
+ // boundary. This mirrors the endsWithDelimiterPrefix encapsulation in
printWithQuotes.
+ final int prefixIndex = isDelimiterStraddleEscapable(delimArray) ?
indexOfDelimiterPrefix(charSeq, delimArray, delimLength) : -1;
+ final int straddleStart = prefixIndex >= 0 ? prefixIndex : end;
while (pos < end) {
char c = charSeq.charAt(pos);
final boolean isDelimiterStart = isDelimiter(c, charSeq, pos,
delimArray, delimLength);
@@ -2417,7 +2445,7 @@ public final class CSVFormat implements Serializable {
final boolean isLf = c == Constants.LF;
// A leading comment marker would be read back as a comment, so
escape it.
final boolean isComment = commentMarkerSet && pos == 0 && c ==
commentChar;
- if (isCr || isLf || c == escape || quoteSet && c == quote ||
isDelimiterStart || isComment) {
+ if (isCr || isLf || c == escape || quoteSet && c == quote ||
isDelimiterStart || isComment || pos >= straddleStart) {
// write out segment up until this char
if (pos > start) {
appendable.append(charSeq, start, pos);
@@ -2463,11 +2491,13 @@ public final class CSVFormat implements Serializable {
final StringBuilder builder = new
StringBuilder(IOUtils.DEFAULT_BUFFER_SIZE);
int c;
boolean firstChar = true;
+ boolean straddling = false;
+ final boolean straddleEscapable =
isDelimiterStraddleEscapable(delimArray);
final char[] lookAheadBuffer = new char[delimLength - 1];
while (EOF != (c = bufferedReader.read())) {
builder.append((char) c);
Arrays.fill(lookAheadBuffer, (char) 0);
- bufferedReader.peek(lookAheadBuffer);
+ final int lookAheadCount = Math.max(0,
bufferedReader.peek(lookAheadBuffer));
// Match the delimiter against the current character plus the
look-ahead buffer only. Rebuilding the test
// string from the whole accumulated builder made this loop O(n^2)
for values without escapable characters.
final String test = String.valueOf((char) c) + new
String(lookAheadBuffer);
@@ -2477,7 +2507,16 @@ public final class CSVFormat implements Serializable {
// A leading comment marker would be read back as a comment, so
escape it.
final boolean isComment = commentMarkerSet && firstChar && c ==
commentChar;
firstChar = false;
- if (isCr || isLf || c == escape || quoteSet && c == quote ||
isDelimiterStart || isComment) {
+ // Once the remaining stream is a straddling prefix of the
delimiter, every remaining character must be
+ // escaped: appended after the bare prefix, the delimiter would
match one character early on read and
+ // shift the field boundary. Such a prefix can only start once the
end of the stream is within look-ahead
+ // range, so a short peek is a precondition. This mirrors the
straddle handling in the CharSequence
+ // overload of printWithEscapes.
+ if (!straddling && straddleEscapable && lookAheadCount <
delimLength - 1 &&
+ indexOfDelimiterPrefix(String.valueOf((char) c) + new
String(lookAheadBuffer, 0, lookAheadCount), delimArray, delimLength) == 0) {
+ straddling = true;
+ }
+ if (isCr || isLf || c == escape || quoteSet && c == quote ||
isDelimiterStart || isComment || straddling) {
// write out segment up until this char
if (pos > start) {
append(builder.substring(start, pos), appendable);
diff --git a/src/main/java/org/apache/commons/csv/Lexer.java
b/src/main/java/org/apache/commons/csv/Lexer.java
index 12f1aa06..83c9c373 100644
--- a/src/main/java/org/apache/commons/csv/Lexer.java
+++ b/src/main/java/org/apache/commons/csv/Lexer.java
@@ -168,6 +168,21 @@ final class Lexer implements Closeable {
return isLastTokenDelimiter;
}
+ /**
+ * Tests whether the given character occurs in the delimiter.
+ *
+ * @param ch the character to test.
+ * @return true if the given character occurs in the delimiter.
+ */
+ private boolean isDelimiterChar(final int ch) {
+ for (final char delimiterChar : delimiter) {
+ if (ch == delimiterChar) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* Tests if the given character indicates the end of the file.
*
@@ -512,8 +527,12 @@ final class Lexer implements Closeable {
case EOF:
throw new CSVException("EOF while processing escape sequence");
default:
- // Now check for meta-characters
- if (isMetaChar(ch)) {
+ // 1) Now check for meta-characters
+ // 2) An escaped delimiter character unescapes to the bare
character: the printer escapes each character of
+ // a multi-character delimiter individually, including a
straddling prefix of the delimiter at the end of
+ // an unquoted value. A fully escaped delimiter never reaches
this method: isEscapeDelimiter() consumes it
+ // first.
+ if (isMetaChar(ch) || isDelimiterChar(ch)) {
return ch;
}
// indicate unexpected char - available from in.getLastChar()
diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java
b/src/test/java/org/apache/commons/csv/CSVParserTest.java
index 50a6ae03..7e86fc00 100644
--- a/src/test/java/org/apache/commons/csv/CSVParserTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java
@@ -1810,7 +1810,8 @@ class CSVParserTest {
final CSVFormat csvFormat =
CSVFormat.DEFAULT.builder().setDelimiter("[|]").setEscape('!').get();
try (CSVParser csvParser = csvFormat.parse(new StringReader(source))) {
CSVRecord csvRecord = csvParser.nextRecord();
- assertEquals("a[|]b![|]c", csvRecord.get(0));
+ // "![" is an escaped delimiter character and unescapes to the
bare "[".
+ assertEquals("a[|]b[|]c", csvRecord.get(0));
assertEquals("xyz", csvRecord.get(1));
csvRecord = csvParser.nextRecord();
assertEquals("abc[abc]", csvRecord.get(0));
@@ -1901,7 +1902,8 @@ class CSVParserTest {
final CSVFormat format =
CSVFormat.DEFAULT.builder().setDelimiter("[|]").setEscape('!').get();
try (CSVParser parser = format.parse(new
StringReader("x![!|!]y![!|"))) {
final CSVRecord record = parser.nextRecord();
- assertEquals("x[|]y![!|", record.get(0));
+ // The truncated "![!|" is not a delimiter; its escaped delimiter
characters unescape individually.
+ assertEquals("x[|]y[|", record.get(0));
assertEquals(1, record.size());
}
}
diff --git a/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
b/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
index f1f910bb..66064f44 100644
--- a/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
@@ -71,6 +71,7 @@ import org.h2.tools.SimpleResultSet;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
/**
@@ -691,6 +692,80 @@ class CSVPrinterTest {
assertEquals("\\\\", sw.toString());
}
+ @Test
+ void testEscapeValueEndingWithDelimiterPrefix() throws IOException {
+ // No quoting available in escape mode, so a value ending in a
straddling prefix of the multi-character
+ // delimiter must have that prefix escaped: appending the bare
delimiter after "a|" yields "a|||", which
+ // reads back with the field boundary shifted one character early.
Mirrors the endsWithDelimiterPrefix
+ // quoting fix for QuoteMode.MINIMAL.
+ final CSVFormat format =
CSVFormat.DEFAULT.builder().setDelimiter("||").setQuote(null).setEscape('\\').get();
+ final StringWriter sw = new StringWriter();
+ try (CSVPrinter printer = new CSVPrinter(sw, format)) {
+ printer.printRecord("a|", "b");
+ printer.printRecord(new StringReader("a|"), new StringReader("b"));
+ // A delimiter prefix in the middle of a value cannot straddle the
appended delimiter and is left alone.
+ printer.printRecord("a|b", "c");
+ }
+ final String string = sw.toString();
+ assertEquals("a\\|||b" + RECORD_SEPARATOR +
+ "a\\|||b" + RECORD_SEPARATOR +
+ "a|b||c" + RECORD_SEPARATOR, string);
+ // The emitted records must read back with the original field
boundaries.
+ try (CSVParser parser = CSVParser.parse(string, format)) {
+ final List<CSVRecord> records = parser.getRecords();
+ assertEquals(3, records.size());
+ assertEquals("a|", records.get(0).get(0));
+ assertEquals("b", records.get(0).get(1));
+ assertEquals("a|", records.get(1).get(0));
+ assertEquals("b", records.get(1).get(1));
+ assertEquals("a|b", records.get(2).get(0));
+ assertEquals("c", records.get(2).get(1));
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ "|||, a||, a\\|\\|",
+ "|||, ||, \\|\\|",
+ "|||, a|, a\\|",
+ "xyxy, axy, a\\x\\y",
+ "xyxy, ax, ax",
+ "[|], a[, a[",
+ "|||, a||z, a||z"
+ })
+ void testEscapeValueEndingWithLongDelimiterPrefix(final String delimiter,
final String value, final String escaped) throws IOException {
+ final CSVFormat format =
CSVFormat.DEFAULT.builder().setDelimiter(delimiter).setQuote(null).setEscape('\\').get();
+ final StringWriter sw = new StringWriter();
+ try (CSVPrinter printer = new CSVPrinter(sw, format)) {
+ printer.printRecord(value, "z");
+ printer.printRecord(new StringReader(value), new
StringReader("z"));
+ }
+ final String expectedRecord = escaped + delimiter + "z" +
RECORD_SEPARATOR;
+ assertEquals(expectedRecord + expectedRecord, sw.toString());
+ try (CSVParser parser = CSVParser.parse(sw.toString(), format)) {
+ final List<CSVRecord> records = parser.getRecords();
+ assertEquals(2, records.size());
+ for (final CSVRecord record : records) {
+ assertArrayEquals(new String[] { value, "z" },
record.values());
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = { "rr", "nn", "tt", "bb", "ff" })
+ void testEscapeValueEndingWithUnescapableDelimiterPrefix(final String
delimiter) throws IOException {
+ // Escaping these letters would turn them into control characters.
Preserve the historical output.
+ final CSVFormat format =
CSVFormat.DEFAULT.builder().setDelimiter(delimiter).setQuote(null).setEscape('\\').get();
+ final String value = "a" + delimiter.charAt(0);
+ final StringWriter sw = new StringWriter();
+ try (CSVPrinter printer = new CSVPrinter(sw, format)) {
+ printer.printRecord(value, "z");
+ printer.printRecord(new StringReader(value), new
StringReader("z"));
+ }
+ final String expectedRecord = value + delimiter + "z" +
RECORD_SEPARATOR;
+ assertEquals(expectedRecord + expectedRecord, sw.toString());
+ }
+
@Test
void testExcelPrintAllArrayOfArrays() throws IOException {
final StringWriter sw = new StringWriter();
diff --git a/src/test/java/org/apache/commons/csv/LexerTest.java
b/src/test/java/org/apache/commons/csv/LexerTest.java
index a76f6e51..3d387f01 100644
--- a/src/test/java/org/apache/commons/csv/LexerTest.java
+++ b/src/test/java/org/apache/commons/csv/LexerTest.java
@@ -436,7 +436,8 @@ class LexerTest {
void testPartialEscapedMultiCharacterDelimiterAtEOF() throws IOException {
final CSVFormat format =
CSVFormat.DEFAULT.builder().setDelimiter("[|]").setEscape('!').get();
try (Lexer lexer = createLexer("x![!|!]y![!|", format)) {
- assertNextToken(EOF, "x[|]y![!|", lexer);
+ // The truncated "![!|" is not a delimiter; its escaped delimiter
characters unescape individually.
+ assertNextToken(EOF, "x[|]y[|", lexer);
}
}
