This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 604b99e2cb [format] Preserve quotes and escapes in unquoted CSV fields
(#10474)
604b99e2cb is described below
commit 604b99e2cba27f34f00cd588df786b7c7cf63223
Author: Zouxxyy <[email protected]>
AuthorDate: Sat Oct 10 14:06:43 2026 +0800
[format] Preserve quotes and escapes in unquoted CSV fields (#10474)
---
.../apache/paimon/format/csv/CsvFormatWriter.java | 5 +-
.../org/apache/paimon/format/csv/CsvParser.java | 40 +++---
.../paimon/format/csv/CsvFileFormatTest.java | 44 ++++++-
.../apache/paimon/format/csv/CsvParserTest.java | 137 +++++++++++++++++++++
4 files changed, 204 insertions(+), 22 deletions(-)
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvFormatWriter.java
b/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvFormatWriter.java
index b9a7d37fff..cd9b950274 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvFormatWriter.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvFormatWriter.java
@@ -140,9 +140,8 @@ public class CsvFormatWriter extends AbstractTextFileWriter
{
return field;
}
- // Only escape if needed. The escape character goes first: CsvParser
drops an escape
- // character that is not followed by a quote or another escape, and
escaping the quotes
- // first would double the escape characters inserted for them.
+ // Escape the escape character first so that quoting the field
preserves it. Escaping
+ // quotes first would double the escape characters inserted for them.
String escaped = field.replace(escape, escape + escape);
return quote + escaped.replace(quote, escape + quote) + quote;
}
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
b/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
index d17b84d3b4..82635c4607 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
@@ -108,30 +108,39 @@ public class CsvParser {
int position = 0;
while (position < line.length() && columnIndex < rowValues.length) {
char c = line.charAt(position);
- if (c == escapeChar) {
- // if the next character is special, process it here as to not
trigger the special
- // handling
- if (inQuotes || inField) {
- int nextCharacter = peekNextCharacter(line, position);
- if (nextCharacter == quoteChar || nextCharacter ==
escapeChar) {
- buffer.append(line.charAt(position + 1));
- position++;
- }
+ // Only a quote at the start of a field enables CSV quoting and
escaping.
+ // Quotes and backslashes in an unquoted field (for example JSON
in TSV)
+ // belong to the field value and must not be interpreted as CSV
syntax.
+ // A single NUL quote or escape is still used by the writer. Only
disable
+ // quoting and escaping when both characters are NUL.
+ if (!inField
+ && buffer.length() == 0
+ && (quoteChar != '\0' || escapeChar != '\0')
+ && c == quoteChar) {
+ rowQuoted[columnIndex] = true;
+ }
+ if (rowQuoted[columnIndex] && c == escapeChar) {
+ int nextCharacter = peekNextCharacter(line, position);
+ if ((inQuotes || inField)
+ && (nextCharacter == quoteChar || nextCharacter ==
escapeChar)) {
+ buffer.append(line.charAt(position + 1));
+ position++;
+ } else {
+ // An escape before an ordinary character is part of the
value.
+ buffer.append(c);
}
- } else if (c == quoteChar) {
+ } else if (rowQuoted[columnIndex] && c == quoteChar) {
// a quote character can be escaped with another quote
character
if ((inQuotes || inField) && peekNextCharacter(line, position)
== quoteChar) {
buffer.append(line.charAt(position + 1));
position++;
} else {
- // the tricky case of an embedded quote in the middle:
a,bc"d"ef,g
- // Embedded quote is not for first 3 characters of the
line, and is not allowed
- // immediately before a separator
+ // Preserve the existing handling of unescaped quotes in
quoted fields.
if (position > 2
&& line.charAt(position - 1) != separatorChar
&& line.length() > (position + 1)
&& line.charAt(position + 1) != separatorChar) {
- // if field starts begins whitespace, skip the
whitespace and quote
+ // Discard whitespace accumulated before an unescaped
quote.
if (buffer.length() != 0 && isAllWhitespace(buffer)) {
buffer.setLength(0);
} else {
@@ -139,9 +148,6 @@ public class CsvParser {
buffer.append(c);
}
}
- if (!inQuotes && buffer.length() == 0) {
- rowQuoted[columnIndex] = true;
- }
inQuotes = !inQuotes;
}
inField = !inField;
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
index 435df580dc..64418583d3 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
@@ -45,6 +45,8 @@ import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.dataformat.csv.Csv
import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.math.BigDecimal;
@@ -383,6 +385,45 @@ public class CsvFileFormatTest extends FormatReadWriteTest
{
}
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testSingleNulQuoteOrEscapeWriteRead(boolean nulQuote) throws
IOException {
+ Options options = new Options();
+ options.set(CsvOptions.QUOTE_CHARACTER, nulQuote ? "\0" : "\"");
+ options.set(CsvOptions.ESCAPE_CHARACTER, nulQuote ? "\\" : "\0");
+ RowType rowType = DataTypes.ROW(DataTypes.STRING(),
DataTypes.STRING(), DataTypes.STRING());
+ List<InternalRow> testData = new ArrayList<>();
+ for (String value :
+ new String[] {
+ "a,b",
+ "say \"hello\"",
+ "C:\\Users\\test\\",
+ "\0start",
+ "middle\0value",
+ "end\0",
+ "plain"
+ }) {
+ testData.add(
+ GenericRow.of(fromString(value), fromString("middle"),
fromString("tail")));
+ testData.add(GenericRow.of(fromString("first"), fromString(value),
fromString("tail")));
+ testData.add(
+ GenericRow.of(fromString("first"), fromString("middle"),
fromString(value)));
+ }
+
+ // Use the real writer and reader: a NUL quote/escape is still emitted
when only
+ // one of the two characters is NUL. Read multiple rows to exercise
parser reuse.
+ List<InternalRow> result =
+ writeThenRead(options, rowType, rowType, testData,
"single_nul_" + nulQuote);
+ assertThat(result).hasSize(testData.size());
+ for (int i = 0; i < testData.size(); i++) {
+ for (int column = 0; column < rowType.getFieldCount(); column++) {
+ assertThat(result.get(i).getString(column))
+ .as("row %s, column %s", i, column)
+ .isEqualTo(testData.get(i).getString(column));
+ }
+ }
+ }
+
@Test
public void testCsvIncludeHeaderWriteRead() throws IOException {
RowType rowType =
@@ -896,8 +937,7 @@ public class CsvFileFormatTest extends FormatReadWriteTest {
@Test
public void testFieldsContainingTheEscapeCharacterRoundTrip() throws
IOException {
RowType rowType = DataTypes.ROW(DataTypes.INT().notNull(),
DataTypes.STRING());
- // every one of these is written unquoted or half-quoted unless the
escape character is
- // itself escaped, and CsvParser then drops it
+ // Quoted fields must escape both quotes and the escape character to
round-trip.
String[] inputs = {
"Special\\Characters", "trailing\\", "a,b\\", "\\\\double",
"\\\"quoteAfterEscape"
};
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvParserTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvParserTest.java
new file mode 100644
index 0000000000..ea2219a7c5
--- /dev/null
+++
b/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvParserTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.format.csv;
+
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import static org.apache.paimon.data.BinaryString.fromString;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests CSV parsing independently of the writer that produces escaped
fields. */
+class CsvParserTest {
+
+ private static final RowType ROW_TYPE =
+ DataTypes.ROW(DataTypes.STRING(), DataTypes.STRING(),
DataTypes.STRING());
+
+ @ParameterizedTest
+ @ValueSource(
+ strings = {
+ "{\"userid\":\"1\",\"memo\":\"a\\nb\"}",
+ "{\"userid\":\"2\",\"memo\":\"C:\\\\Users\\\\test\"}",
+ "{\"userid\":\"3\",\"memo\":\"a\\tb\"}",
+ "{\"userid\":\"4\",\"memo\":\"a\\\"b\"}",
+ "{\"userid\":\"5\",\"memo\":\"\\u871c\\u96ea\"}",
+ "{\"userid\":\"6\",\"memo\":\"\"}",
+ "unquoted\\path\\",
+ "\\\\server\\path",
+ "embedded\"quote",
+ "doubled\"\"quotes",
+ "leading \\\"quote",
+ " \"value\"",
+ "\\\"leading"
+ })
+ void testUnquotedFieldsPreserveQuotesAndEscapes(String value) {
+ CsvParser parser = parser("\t", "\"", "\\");
+ // Exercise each column and reuse the parser across rows.
+ assertThat(parser.parse(value +
"\tsecond\tlast")).isEqualTo(row(value, "second", "last"));
+ assertThat(parser.parse("first\t" + value + "\tlast"))
+ .isEqualTo(row("first", value, "last"));
+ assertThat(parser.parse("first\tsecond\t" + value))
+ .isEqualTo(row("first", "second", value));
+ }
+
+ @Test
+ void testQuotedFieldsDecodeEscapesAndContainDelimiters() {
+ CsvParser parser = parser(",", "\"", "\\");
+ assertThat(parser.parse("\"a,b\",\"C:\\\\Users\\\\test\",\"say
\\\"hello\\\"\""))
+ .isEqualTo(row("a,b", "C:\\Users\\test", "say \"hello\""));
+ // Preserve Paimon's existing doubled-quote behavior; Spark's default
differs here.
+ assertThat(parser.parse("\"a\"\"b\",unquoted\\path,\"c\\\\d\""))
+ .isEqualTo(row("a\"b", "unquoted\\path", "c\\d"));
+ }
+
+ @Test
+ void testUnknownEscapesInQuotedFieldsArePreserved() {
+ assertThat(parser(",", "\"",
"\\").parse("\"a\\nb\",\"\\u871c\",\"path\\q\""))
+ .isEqualTo(row("a\\nb", "\\u871c", "path\\q"));
+ }
+
+ @Test
+ void testEmptyQuotedFieldsResetQuotingState() {
+ CsvParser parser = parser(",", "\"", "\\");
+ assertThat(parser.parse("\"\",plain\\value,\"\"")).isEqualTo(row("",
"plain\\value", ""));
+ assertThat(parser.parse("\"first\",,last\\value"))
+ .isEqualTo(GenericRow.of(fromString("first"), null,
fromString("last\\value")));
+ }
+
+ @Test
+ void testEmbeddedQuoteDoesNotProtectDelimiter() {
+ assertThat(parser(",", "\"", "\\").parse("a\"b,c\"d,last"))
+ .isEqualTo(row("a\"b", "c\"d", "last"));
+ }
+
+ @Test
+ void testCustomQuoteAndEscapeCharacters() {
+ CsvParser parser = parser("|", "'", "/");
+ assertThat(parser.parse("raw//path/'quote|'a/'b//c'|'x|y'"))
+ .isEqualTo(row("raw//path/'quote", "a'b/c", "x|y"));
+ }
+
+ @Test
+ void testDisabledQuotingAndEscapingPreserveNullCharacters() {
+ assertThat(parser(",", "\0", "\0").parse("a\0b,\0start,end\0"))
+ .isEqualTo(row("a\0b", "\0start", "end\0"));
+ }
+
+ @Test
+ void testUnterminatedQuotedFieldRemainsNull() {
+ assertThat(parser(",", "\"",
"\\").parse("first,second,\"unterminated"))
+ .isEqualTo(GenericRow.of(fromString("first"),
fromString("second"), null));
+ }
+
+ @Test
+ void testQuotedNullLiteralWithProjectionAndParserReuse() {
+ Options options = new Options();
+ options.set(CsvOptions.NULL_LITERAL, "NULL");
+ CsvParser parser = new CsvParser(ROW_TYPE, new int[] {1, 0, 2}, new
CsvOptions(options));
+ assertThat(parser.parse("NULL,\"NULL\",\"\""))
+ .isEqualTo(GenericRow.of(fromString("NULL"), null,
fromString("")));
+ assertThat(parser.parse("\"NULL\",NULL,plain\\value"))
+ .isEqualTo(GenericRow.of(null, fromString("NULL"),
fromString("plain\\value")));
+ }
+
+ private static CsvParser parser(String delimiter, String quote, String
escape) {
+ Options options = new Options();
+ options.set(CsvOptions.FIELD_DELIMITER, delimiter);
+ options.set(CsvOptions.QUOTE_CHARACTER, quote);
+ options.set(CsvOptions.ESCAPE_CHARACTER, escape);
+ return new CsvParser(ROW_TYPE, new int[] {0, 1, 2}, new
CsvOptions(options));
+ }
+
+ private static GenericRow row(String first, String second, String third) {
+ return GenericRow.of(fromString(first), fromString(second),
fromString(third));
+ }
+}
