This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 07c40ba17a GH-39961: [C++][Python] Propagate CSV parse delimiter to
write options (#49858)
07c40ba17a is described below
commit 07c40ba17ad01ff5557aab0f8e0717e13eca473a
Author: egolearner <[email protected]>
AuthorDate: Tue Jul 21 08:30:26 2026 +0800
GH-39961: [C++][Python] Propagate CSV parse delimiter to write options
(#49858)
### Rationale for this change
resolve #39961
### What changes are included in this PR?
When writing a dataset with CsvFileFormat configured with a custom
ParseOptions delimiter (e.g. delimiter=">"), the output CSV still used the
default "," delimiter. This was because:
- C++ CsvFileFormat::DefaultWriteOptions() always created
WriteOptions::Defaults() with delimiter=',', ignoring the
parse_options.delimiter stored on the format object.
- Python CsvFileFormat.make_write_options() unconditionally overwrote the
C++ write options with a fresh WriteOptions(**kwargs), discarding any C++-side
default that might have been set.
Fix the C++ side by propagating parse_options.delimiter into the write
options in DefaultWriteOptions(). Fix the Python side by reading the delimiter
from the C++ default when the caller does not explicitly specify one,
preventing the overwrite from losing the propagated value.
### Are these changes tested?
Yes
### Are there any user-facing changes?
No
* GitHub Issue: #39961
Lead-authored-by: egolearner <[email protected]>
Co-authored-by: Copilot Autofix powered by AI
<[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
cpp/src/arrow/dataset/file_csv.cc | 1 +
cpp/src/arrow/dataset/file_csv_test.cc | 16 +++++++++++++
python/pyarrow/_dataset.pyx | 12 +++++++++-
python/pyarrow/tests/test_dataset.py | 41 ++++++++++++++++++++++++++++++++++
4 files changed, 69 insertions(+), 1 deletion(-)
diff --git a/cpp/src/arrow/dataset/file_csv.cc
b/cpp/src/arrow/dataset/file_csv.cc
index cede268107..c0d85581f6 100644
--- a/cpp/src/arrow/dataset/file_csv.cc
+++ b/cpp/src/arrow/dataset/file_csv.cc
@@ -474,6 +474,7 @@ std::shared_ptr<FileWriteOptions>
CsvFileFormat::DefaultWriteOptions() {
new CsvFileWriteOptions(shared_from_this()));
csv_options->write_options =
std::make_shared<csv::WriteOptions>(csv::WriteOptions::Defaults());
+ csv_options->write_options->delimiter = parse_options.delimiter;
return csv_options;
}
diff --git a/cpp/src/arrow/dataset/file_csv_test.cc
b/cpp/src/arrow/dataset/file_csv_test.cc
index e8e5838e6f..9ccc1140ad 100644
--- a/cpp/src/arrow/dataset/file_csv_test.cc
+++ b/cpp/src/arrow/dataset/file_csv_test.cc
@@ -449,6 +449,22 @@ TEST_P(TestCsvFileFormat,
WriteRecordBatchReaderCustomOptions) {
ASSERT_EQ("0\n0\n0\n0\n0\n", written->ToString());
}
+TEST_P(TestCsvFileFormat, WriteRecordBatchReaderDelimiterPropagation) {
+ // Verify that CsvFileFormat::DefaultWriteOptions() propagates the parse
+ // delimiter to write options
+ auto csv_format = std::make_shared<CsvFileFormat>();
+ csv_format->parse_options.delimiter = '|';
+ auto write_options =
+
checked_pointer_cast<CsvFileWriteOptions>(csv_format->DefaultWriteOptions());
+ ASSERT_EQ(write_options->write_options->delimiter, '|');
+
+ // Verify that the default delimiter is ',' when no parse options are set
+ auto default_format = std::make_shared<CsvFileFormat>();
+ auto default_write_options =
+
checked_pointer_cast<CsvFileWriteOptions>(default_format->DefaultWriteOptions());
+ ASSERT_EQ(default_write_options->write_options->delimiter, ',');
+}
+
TEST_P(TestCsvFileFormat, CountRows) { TestCountRows(); }
TEST_P(TestCsvFileFormat, FragmentEquals) { TestFragmentEquals(); }
diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx
index 66d2cba27a..d40614a61f 100644
--- a/python/pyarrow/_dataset.pyx
+++ b/python/pyarrow/_dataset.pyx
@@ -2255,7 +2255,17 @@ cdef class CsvFileFormat(FileFormat):
"""
cdef CsvFileWriteOptions opts = \
<CsvFileWriteOptions> FileFormat.make_write_options(self)
- opts.write_options = WriteOptions(**kwargs)
+ # Start from the C++ defaults, which carry over fields from the
+ # format's parse_options (e.g. the delimiter), and apply caller
+ # overrides on top instead of replacing the WriteOptions object
+ # and discarding those defaults. None is treated as "unspecified"
+ # to match the previous WriteOptions(**kwargs) semantics.
+ write_options = opts.write_options
+ for key, value in kwargs.items():
+ if value is None:
+ continue
+ setattr(write_options, key, value)
+ opts.write_options = write_options
return opts
@property
diff --git a/python/pyarrow/tests/test_dataset.py
b/python/pyarrow/tests/test_dataset.py
index 63c537eae5..b12a1bce60 100644
--- a/python/pyarrow/tests/test_dataset.py
+++ b/python/pyarrow/tests/test_dataset.py
@@ -4987,6 +4987,47 @@ def test_write_dataset_csv(tempdir):
assert result.equals(table)
+def test_csv_make_write_options_uses_parse_delimiter():
+ # CsvFileFormat.make_write_options propagates the parse delimiter
+ # to write options when no explicit delimiter is given
+ csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter="|"))
+ write_opts = csv_format.make_write_options()
+ assert write_opts.write_options.delimiter == "|"
+
+ # delimiter=None is treated as "unspecified" and the propagated
+ # value is preserved (matches WriteOptions(**kwargs) semantics)
+ write_opts = csv_format.make_write_options(delimiter=None)
+ assert write_opts.write_options.delimiter == "|"
+
+ # An explicitly passed delimiter takes precedence
+ csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">"))
+ write_opts = csv_format.make_write_options(delimiter="|")
+ assert write_opts.write_options.delimiter == "|"
+
+ # The default delimiter is still "," when no parse options are given
+ csv_format = ds.CsvFileFormat()
+ write_opts = csv_format.make_write_options()
+ assert write_opts.write_options.delimiter == ","
+
+
+def test_write_dataset_uses_csv_parse_delimiter(tempdir):
+ table = pa.table({
+ "B": ["B1", "B2"],
+ "C": ["C1", "C2"],
+ })
+
+ csv_format = ds.CsvFileFormat(pa.csv.ParseOptions(delimiter=">"))
+ ds.write_dataset(table, tempdir, format=csv_format)
+
+ with open(tempdir / "part-0.csv") as fh:
+ content = fh.read()
+ assert content == '"B">"C"\n"B1">"C1"\n"B2">"C2"\n'
+
+ # Roundtrip: reading back with the same delimiter recovers the table
+ result = ds.dataset(tempdir, format=csv_format).to_table()
+ assert result.equals(table)
+
+
@pytest.mark.parquet
def test_write_dataset_parquet_file_visitor(tempdir):
table = pa.table([
