https://github.com/python/cpython/commit/e87da800b6103fd890841688da48e41e211be7b9 commit: e87da800b6103fd890841688da48e41e211be7b9 branch: 3.14 author: Miss Islington (bot) <[email protected]> committer: gpshead <[email protected]> date: 2026-09-09T16:15:49Z summary:
[3.14] gh-123018: Keep the libedit history file header when truncating (GH-157165) (#157173) gh-123018: Keep the libedit history file header when truncating (GH-157165) libedit's history_truncate_file() keeps the last N lines of the file, which drops the "_HiStOrY_V2_" header line that its own write_history() emits and that its read_history() requires. So on a libedit build, readline.write_history_file() or readline.append_history_file() after readline.set_history_length() produced a file that readline.read_history_file() rejected with EINVAL. Under the libedit emulation, truncate the file ourselves and keep the header, resolving the default "~/.history" the same way libedit does. Apple's libedit fork already preserves the header, so the workaround is not compiled on macOS. (cherry picked from commit 26387858990674f8cd20566f89c700fa1de5d829) Co-authored-by: Gregory P. Smith <[email protected]> files: A Misc/NEWS.d/next/Library/2026-09-08-04-10-00.gh-issue-123018.hV2kQe.rst M Lib/test/test_readline.py M Modules/readline.c diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 45192fe508270dd..4a630ab35c648d4 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -171,6 +171,44 @@ def test_write_read_limited_history(self): # Readline seems to report an additional history element. self.assertIn(readline.get_current_history_length(), (2, 3)) + def test_write_read_zero_length_history(self): + previous_length = readline.get_history_length() + self.addCleanup(readline.set_history_length, previous_length) + + readline.clear_history() + readline.add_history("first line") + readline.set_history_length(0) + readline.write_history_file(TESTFN) + self.addCleanup(os.remove, TESTFN) + + readline.clear_history() + # libedit cannot read an empty history file, only one that still + # has its header line. How many items remain is not checked: + # libedit's own history_truncate_file() ignores a length of 0. + readline.read_history_file(TESTFN) + + @unittest.skipUnless(hasattr(readline, "append_history_file"), + "append_history not available") + def test_append_limited_history(self): + previous_length = readline.get_history_length() + self.addCleanup(readline.set_history_length, previous_length) + + readline.clear_history() + readline.add_history("first line") + readline.add_history("second line") + readline.write_history_file(TESTFN) + self.addCleanup(os.remove, TESTFN) + + readline.add_history("third line") + readline.set_history_length(2) + readline.append_history_file(1, TESTFN) + + readline.clear_history() + readline.read_history_file(TESTFN) + self.assertEqual(readline.get_history_item(1), "second line") + self.assertEqual(readline.get_history_item(2), "third line") + self.assertEqual(readline.get_history_item(3), None) + class TestReadline(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2026-09-08-04-10-00.gh-issue-123018.hV2kQe.rst b/Misc/NEWS.d/next/Library/2026-09-08-04-10-00.gh-issue-123018.hV2kQe.rst new file mode 100644 index 000000000000000..d75aa6950eb0711 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-08-04-10-00.gh-issue-123018.hV2kQe.rst @@ -0,0 +1,6 @@ +Fix :func:`readline.write_history_file` and +:func:`readline.append_history_file` producing a history file that +:func:`readline.read_history_file` could not load when a limit was set with +:func:`readline.set_history_length` and Python was built against +``libedit``. This works around `NetBSD PR 60322 +<https://gnats.netbsd.org/60322>`_. diff --git a/Modules/readline.c b/Modules/readline.c index 7708f47d4d03279..a7835d6bdb9f37c 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -10,16 +10,27 @@ /* Standard definitions */ #include "Python.h" +#include "pycore_fileutils.h" // _Py_open_noraise() #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_pylifecycle.h" // _Py_SetLocaleFromEnv() #include <errno.h> // errno +#include <pwd.h> // getpwuid_r() #include <signal.h> // SIGWINCH #include <stdlib.h> // free() #include <string.h> // strdup() +#ifdef HAVE_FCNTL_H +# include <fcntl.h> // O_RDWR +#endif #ifdef HAVE_SYS_SELECT_H # include <sys/select.h> // select() #endif +#ifdef HAVE_SYS_STAT_H +# include <sys/stat.h> // fstat() +#endif +#ifdef HAVE_UNISTD_H +# include <unistd.h> // ftruncate() +#endif #if defined(HAVE_SETLOCALE) /* GNU readline() mistakenly sets the LC_CTYPE locale. @@ -319,6 +330,176 @@ readline_read_history_file_impl(PyObject *module, PyObject *filename_obj) static int _history_length = -1; /* do not truncate history by default */ +#ifndef __APPLE__ +/* macOS libedit carries a patch fixing the bug this works around. */ +/* Return libedit's default history file, "~/.history". This must resolve + * the same file as libedit's private _default_history_file() + * (https://cvsweb.netbsd.org/bsdweb.cgi/src/lib/libedit/readline.c) so that we + * truncate the same file write_history() and append_history() just wrote: the + * home directory comes from the password database, $HOME is not consulted. + * The result must be freed with PyMem_RawFree(). */ +static char * +_py_libedit_default_history_file(void) +{ + struct passwd *pw = NULL; + char *path = NULL; +#ifdef HAVE_GETPWUID_R + struct passwd pwd; + char *buf = NULL; + Py_ssize_t bufsize = sysconf(_SC_GETPW_R_SIZE_MAX); + if (bufsize == -1) { + bufsize = 1024; + } + for (;;) { + char *newbuf = PyMem_RawRealloc(buf, bufsize); + if (newbuf == NULL) { + break; + } + buf = newbuf; + int status = getpwuid_r(getuid(), &pwd, buf, bufsize, &pw); + if (status == 0) { + break; + } + pw = NULL; + if (status != ERANGE || bufsize > (PY_SSIZE_T_MAX >> 1)) { + break; + } + bufsize <<= 1; + } +#else + pw = getpwuid(getuid()); +#endif + if (pw != NULL && pw->pw_dir != NULL) { + size_t len = strlen(pw->pw_dir) + sizeof("/.history"); + path = PyMem_RawMalloc(len); + if (path != NULL) { + PyOS_snprintf(path, len, "%s/.history", pw->pw_dir); + } + } +#ifdef HAVE_GETPWUID_R + PyMem_RawFree(buf); +#endif + return path; +} + +/* libedit's history_truncate_file() keeps the last nlines lines of the + * file, which drops the "_HiStOrY_V2_" header line that its own + * write_history() emits and that its read_history() requires: once + * truncated, the file can no longer be loaded (gh-123018). Truncate the + * file ourselves and keep the header. + * + * Upstream libedit (NetBSD, and the portable releases from thrysoee.dk) + * still has this bug, reported as https://gnats.netbsd.org/60322. Apple's + * libedit fork patches its history_truncate_file() to copy the first line, + * so this workaround is not compiled on macOS. We only keep the header + * when it is actually there, so a file without one (e.g. written by GNU + * readline) is not given a bogus header. + * + * History files are small, so the whole file is read into memory and + * rewritten in place, as libedit does. GNU readline instead writes a + * temporary file and renames it over the original. */ +static int +_py_libedit_history_truncate_file(const char *filename, int nlines) +{ + static const char cookie[] = "_HiStOrY_V2_\n"; + const size_t cookie_len = sizeof(cookie) - 1; + char *default_file = NULL; + char *buf = NULL; + FILE *fp = NULL; + int ret = -1; + + if (filename == NULL) { + default_file = _py_libedit_default_history_file(); + if (default_file == NULL) { + return -1; + } + filename = default_file; + } + int fd = _Py_open_noraise(filename, O_RDWR); + if (fd < 0) { + goto done; + } + fp = fdopen(fd, "r+"); + if (fp == NULL) { + close(fd); + goto done; + } + struct stat st; + if (fstat(fd, &st) != 0) { + goto done; + } + size_t size = (size_t)st.st_size; + buf = PyMem_RawMalloc(size); + if (buf == NULL || fread(buf, 1, size, fp) != size) { + goto done; + } + + size_t header = 0; + if (size >= cookie_len && memcmp(buf, cookie, cookie_len) == 0) { + header = cookie_len; + } + const char *end = buf + size; + + /* Count the lines following the header. */ + size_t total = 0; + for (const char *p = buf + header; + (p = memchr(p, '\n', end - p)) != NULL; + p++) { + total++; + } + if (end > buf + header && end[-1] != '\n') { + total++; /* an unterminated last line */ + } + if (total <= (size_t)nlines) { + ret = 0; /* nothing to drop */ + goto done; + } + + /* Skip the leading lines, keeping the header and the last nlines. */ + const char *tail = buf + header; + for (size_t skip = total - (size_t)nlines; skip > 0; skip--) { + const char *nl = memchr(tail, '\n', end - tail); + if (nl == NULL) { + tail = end; /* the unterminated last line is dropped too */ + break; + } + tail = nl + 1; + } + size_t taillen = end - tail; + if (fseek(fp, 0, SEEK_SET) != 0 + || fwrite(buf, 1, header, fp) != header + || fwrite(tail, 1, taillen, fp) != taillen + || fflush(fp) != 0 + || ftruncate(fd, (off_t)(header + taillen)) != 0) { + goto done; + } + ret = 0; + +done: + if (fp != NULL) { + fclose(fp); + } + PyMem_RawFree(buf); + PyMem_RawFree(default_file); + return ret; +} +#endif /* !__APPLE__ */ + +/* Truncate the history file after write_history() or append_history(). + * Like the history_truncate_file() calls this replaces, failures are + * ignored by the callers: the history was already saved successfully and + * a failed truncation only leaves the file longer than requested. */ +static int +_py_history_truncate_file(const char *filename, int nlines) +{ +#ifndef __APPLE__ + if (using_libedit_emulation) { + return _py_libedit_history_truncate_file(filename, nlines); + } +#endif + return history_truncate_file(filename, nlines); +} + /* Exported function to save a readline history file */ /*[clinic input] @@ -360,7 +541,7 @@ readline_write_history_file_impl(PyObject *module, PyObject *filename_obj) errno = err = write_history(filename); int history_length = FT_ATOMIC_LOAD_INT_RELAXED(_history_length); if (!err && history_length >= 0) - history_truncate_file(filename, history_length); + _py_history_truncate_file(filename, history_length); Py_XDECREF(filename_bytes); errno = err; if (errno) @@ -419,7 +600,7 @@ readline_append_history_file_impl(PyObject *module, int nelements, nelements - libedit_append_replace_history_offset, filename); int history_length = FT_ATOMIC_LOAD_INT_RELAXED(_history_length); if (!err && history_length >= 0) - history_truncate_file(filename, history_length); + _py_history_truncate_file(filename, history_length); Py_XDECREF(filename_bytes); errno = err; if (errno) _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
