https://github.com/python/cpython/commit/a60343ed17785ebbcd43de9080cadd8e2541db6f commit: a60343ed17785ebbcd43de9080cadd8e2541db6f branch: main author: Serhiy Storchaka <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-09-13T09:49:39Z summary:
gh-156955: Speed up csv.writer by caching the set of special characters (GH-157298) Cache in the dialect a 128-bit set of ASCII characters which need quoting or escaping (delimiter, quotechar, escapechar, '\r', '\n' and characters of lineterminator) and a flag whether any of them is non-ASCII. Testing a character is now one bit test instead of five comparisons and a call to PyUnicode_FindChar(). Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> files: A Misc/NEWS.d/next/Library/2026-09-11-12-00-00.gh-issue-156955.bLtmAp.rst M Lib/test/test_csv.py M Modules/_csv.c diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 73e282d1abf717..36fa7e3572a5ac 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -227,6 +227,18 @@ def test_write_quoting(self): quoting = csv.QUOTE_STRINGS) self._write_test(['a','',None,1], '"a","",,"1"', quoting = csv.QUOTE_NOTNULL) + # FULLWIDTH QUOTATION MARK + self._write_test(['a', 1, 'p,q', 'r＂s', 'x！y'], + 'a,1,＂p,q＂,＂r＂＂s＂,x！y', + quotechar='＂') + + def test_write_delimiter(self): + self._write_test(['a', 1, 'p,q', 'x;y'], 'a,1,"p,q",x;y') + self._write_test(['a', 1, 'p;q', 'x,y'], 'a;1;"p;q";x,y', delimiter=';') + self._write_test(['a', 1, 'p\0q', 'x,y'], 'a\x001\0"p\0q"\0x,y', + delimiter='\0') + self._write_test(['a', 1, 'p🍌q', 'x🍍y'], 'a🍌1🍌"p🍌q"🍌x🍍y', + delimiter='🍌') def test_write_escape(self): self._write_test(['a',1,'p,q'], 'a,1,"p,q"', @@ -258,19 +270,26 @@ def test_write_escape(self): escapechar='\\', quoting=csv.QUOTE_MINIMAL) self._write_test(['C\\', '6', '7', 'X"'], 'C\\\\,6,7,"X"""', escapechar='\\', quoting=csv.QUOTE_MINIMAL) + # SYMBOL FOR ESCAPE + self._write_test(['a', 1, 'p,q', 'r\u241bs', 'x\u241ay'], + 'a,1,p\u241b,q,r\u241b\u241bs,x\u241ay', + escapechar='\u241b', quoting=csv.QUOTE_NONE) def test_write_lineterminator(self): - for lineterminator in '\r\n', '\n', '\r', '!@#', '\0': + for lineterminator in ('\r\n', '\n', '\r', '!@#', '\0', + '\x85', '\u2028', '\U0001f600'): with self.subTest(lineterminator=lineterminator): with StringIO() as sio: writer = csv.writer(sio, lineterminator=lineterminator) writer.writerow(['a', 'b']) writer.writerow([1, 2]) writer.writerow(['\r', '\n']) + writer.writerow([f'a{lineterminator[-1]}b', 'c']) self.assertEqual(sio.getvalue(), f'a,b{lineterminator}' f'1,2{lineterminator}' - f'"\r","\n"{lineterminator}') + f'"\r","\n"{lineterminator}' + f'"a{lineterminator[-1]}b",c{lineterminator}') def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') diff --git a/Misc/NEWS.d/next/Library/2026-09-11-12-00-00.gh-issue-156955.bLtmAp.rst b/Misc/NEWS.d/next/Library/2026-09-11-12-00-00.gh-issue-156955.bLtmAp.rst new file mode 100644 index 00000000000000..2a452a2a6e1652 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-11-12-00-00.gh-issue-156955.bLtmAp.rst @@ -0,0 +1,3 @@ +Speed up :func:`csv.writer` by caching the set of characters that need +quoting or escaping in the dialect. Writing long fields is now up to 5 times +faster. diff --git a/Modules/_csv.c b/Modules/_csv.c index c640f2d36a8464..6af66c3f09a03b 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -117,7 +117,12 @@ typedef struct { Py_UCS4 quotechar; /* quote character */ Py_UCS4 escapechar; /* escape character */ PyObject *lineterminator; /* string to write between records */ - + /* Cache for the writer: bit c is set if the ASCII character c needs + quoting or escaping (delimiter, quotechar, escapechar, '\r', '\n' + and the characters of lineterminator). */ + uint64_t special_chars[2]; + /* Whether any of the special characters is non-ASCII. */ + bool nonascii_special; } DialectObj; typedef struct { @@ -332,6 +337,54 @@ _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt) return 0; } +static void +dialect_add_special_char(DialectObj *self, Py_UCS4 c) +{ + if (c == NOT_SET) { + return; + } + if (c < 128) { + self->special_chars[c / 64] |= (uint64_t)1 << (c % 64); + } + else { + self->nonascii_special = true; + } +} + +static void +dialect_init_special_chars_cache(DialectObj *self) +{ + self->special_chars[0] = self->special_chars[1] = 0; + self->nonascii_special = false; + dialect_add_special_char(self, self->delimiter); + dialect_add_special_char(self, self->quotechar); + dialect_add_special_char(self, self->escapechar); + dialect_add_special_char(self, '\r'); + dialect_add_special_char(self, '\n'); + PyObject *lt = self->lineterminator; + for (Py_ssize_t i = 0; i < PyUnicode_GET_LENGTH(lt); i++) { + dialect_add_special_char(self, PyUnicode_READ_CHAR(lt, i)); + } +} + +/* Whether the character needs quoting or escaping by the writer. */ +static inline bool +dialect_is_special_char(DialectObj *self, Py_UCS4 c) +{ + if (c < 128) { + return (self->special_chars[c / 64] >> (c % 64)) & 1; + } + if (!self->nonascii_special) { + return false; + } + return (c == self->delimiter || + c == self->quotechar || + c == self->escapechar || + PyUnicode_FindChar(self->lineterminator, c, 0, + PyUnicode_GET_LENGTH(self->lineterminator), + 1) >= 0); +} + static int dialect_check_quoting(int quoting) { @@ -558,6 +611,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { goto err; } + dialect_init_special_chars_cache(self); ret = Py_NewRef(self); err: @@ -1208,14 +1262,7 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, Py_UCS4 c = PyUnicode_READ(field_kind, field_data, i); int want_escape = 0; - if (c == dialect->delimiter || - c == dialect->escapechar || - c == dialect->quotechar || - c == '\n' || - c == '\r' || - PyUnicode_FindChar( - dialect->lineterminator, c, 0, - PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0) { + if (dialect_is_special_char(dialect, c)) { if (dialect->quoting == QUOTE_NONE) want_escape = 1; else { _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
