https://github.com/python/cpython/commit/e0c28e2a00746e9100969868a71f0e5524450ded commit: e0c28e2a00746e9100969868a71f0e5524450ded branch: 3.14 author: Pablo Galindo Salgado <[email protected]> committer: pablogsal <[email protected]> date: 2026-10-05T10:24:22Z summary:
[3.14] Add MSan to CI (GH-158625) (#158834) * Run a job that the test suite with MSan to the CI (#158625) * Run the test suite with MSan in CI * Additional fixes * Add `_Py_MSAN_UNPOISON_STRING` * Apply Victor's suggestions Co-authored-by: Victor Stinner <[email protected]> * Apply Victor's suggestions Co-authored-by: Victor Stinner <[email protected]> --------- Co-authored-by: Victor Stinner <[email protected]> (cherry picked from commit b93fb19a6e3118857d9a1dc4ffcc179b632a88c1) * Mark bytes returned by getrandom as initialized for MSan * Handle invalid non-ASCII struct formats in the fuzz harness --------- Co-authored-by: Stan Ulbrych <[email protected]> Co-authored-by: Victor Stinner <[email protected]> files: M .github/workflows/build.yml M .github/workflows/reusable-san.yml M Doc/using/configure.rst M Include/pyport.h M Lib/test/test_faulthandler.py M Lib/test/test_xxtestfuzz.py M Modules/_testinternalcapi.c M Modules/_xxtestfuzz/fuzzer.c M Modules/posixmodule.c M Modules/socketmodule.c M Python/instrumentation.c M configure M configure.ac diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4760670cffc93bf..58bcaf86258ab21 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -589,6 +589,9 @@ jobs: - check-name: Undefined behavior sanitizer: UBSan free-threading: false + - check-name: Memory + sanitizer: MSan + free-threading: false uses: ./.github/workflows/reusable-san.yml with: sanitizer: ${{ matrix.sanitizer }} diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 317ac892f4b05a4..d4429b98d211dd0 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -67,13 +67,26 @@ jobs: || '' }} - name: UBSan option setup - if: inputs.sanitizer != 'TSan' + if: inputs.sanitizer == 'UBSan' run: >- echo "UBSAN_OPTIONS=${SAN_LOG_OPTION}" >> "$GITHUB_ENV" env: SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + - name: MSan option setup + if: inputs.sanitizer == 'MSan' + run: | + echo "MSAN_OPTIONS=${SAN_LOG_OPTION} allocator_may_return_null=1 handle_segv=0" >> "$GITHUB_ENV" + # MSan reports false positives for memory initialized by libraries + # that are not built with MSan, so disable modules that use them. + { + echo '*disabled*' + echo '_bz2 _ctypes _curses _curses_panel _dbm _decimal _gdbm _hashlib' + echo '_lzma _sqlite3 _ssl _tkinter _uuid _zstd readline zlib' + } > Modules/Setup.local + env: + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" @@ -98,6 +111,8 @@ jobs: # gh-157958: -O2 instead of the pydebug default -Og to avoid a clang 21 # compile-time blowup on some interpreter files. # (https://github.com/llvm/llvm-project/issues/179695) + # MSan uses --with-assertions instead of --with-pydebug because its + # hooks on the Python memory allocators hide uninitialized reads. - name: Configure CPython run: >- ./configure @@ -106,9 +121,11 @@ jobs: ${{ inputs.sanitizer == 'TSan' && '--with-thread-sanitizer' + || inputs.sanitizer == 'MSan' + && '--with-memory-sanitizer' || '--with-undefined-behavior-sanitizer' }} - --with-pydebug + ${{ inputs.sanitizer == 'MSan' && '--with-assertions' || '--with-pydebug' }} ${{ inputs.sanitizer == 'TSan' && '--with-openssl="$OPENSSL_DIR" --with-openssl-rpath=auto' || '' }} ${{ inputs.free-threading && '--disable-gil' || '' }} - name: Build CPython diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index e190c85920c7b9e..c038e4bc340e52a 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -920,6 +920,10 @@ Debug options Enable MemorySanitizer allocation error detector, ``msan`` (default is no). + MSan reports false positives for memory initialized by libraries that are + not built with MSan, so either build all dependencies with MSan or disable + the extension modules that use them in :file:`Modules/Setup.local`. + .. versionadded:: 3.6 .. option:: --with-undefined-behavior-sanitizer diff --git a/Include/pyport.h b/Include/pyport.h index 2c8567f2554d9b9..22820884b45fb70 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -577,6 +577,8 @@ extern "C" { # if !defined(_Py_MEMORY_SANITIZER) # define _Py_MEMORY_SANITIZER # define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) +# define _Py_MSAN_UNPOISON(PTR, SIZE) (__msan_unpoison(PTR, SIZE)) +# define _Py_MSAN_UNPOISON_STRING(STR) (__msan_unpoison_string(STR)) # endif # endif # if __has_feature(address_sanitizer) @@ -615,6 +617,12 @@ extern "C" { #ifndef _Py_NO_SANITIZE_MEMORY # define _Py_NO_SANITIZE_MEMORY #endif +#ifndef _Py_MSAN_UNPOISON +# define _Py_MSAN_UNPOISON(PTR, SIZE) +#endif +#ifndef _Py_MSAN_UNPOISON_STRING +# define _Py_MSAN_UNPOISON_STRING(STR) +#endif /* AIX has __bool__ redefined in it's system header file. */ #if defined(_AIX) && defined(__bool__) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 9f4dbe7f8663475..bbf4543332822b4 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -34,8 +34,8 @@ def skip_if_sanitizer_signal(signame): - return support.skip_if_sanitizer(f"TSAN/UBSan itercepts {signame}", - thread=True, ub=True) + return support.skip_if_sanitizer(f"TSan/UBSan/MSan intercepts {signame}", + thread=True, ub=True, memory=True) def expected_traceback(lineno1, lineno2, header, min_count=1): diff --git a/Lib/test/test_xxtestfuzz.py b/Lib/test/test_xxtestfuzz.py index 3304c6e703a1736..2cbce4938748db9 100644 --- a/Lib/test/test_xxtestfuzz.py +++ b/Lib/test/test_xxtestfuzz.py @@ -18,6 +18,7 @@ def test_sample_input_smoke_test(self): _xxtestfuzz.run(b"1") _xxtestfuzz.run(b"AAAAAAA") _xxtestfuzz.run(b"AAAAAA\0") + _xxtestfuzz.run(b"\xff\0") if __name__ == "__main__": diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 6df5032205e645e..4a9ef56e3fcd003 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1489,8 +1489,8 @@ check_pyobject_forbidden_bytes_is_freed(PyObject *self, static PyObject * check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) { - /* ASan or TSan would report an use-after-free error */ -#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) + /* ASan, MSan or TSan would report an error. */ +#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) || defined(_Py_MEMORY_SANITIZER) Py_RETURN_NONE; #else PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index f00f8b3be9d107e..44660085911f988 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -133,6 +133,10 @@ static int fuzz_struct_unpack(const char* data, size_t size) { if (unpacked == NULL && PyErr_ExceptionMatches(PyExc_SystemError)) { PyErr_Clear(); } + /* Ignore any ValueError, these are triggered by non-ASCII format. */ + if (unpacked == NULL && PyErr_ExceptionMatches(PyExc_ValueError)) { + PyErr_Clear(); + } /* Ignore any struct.error exceptions, these can be caused by invalid formats or incomplete buffers both of which are common. */ if (unpacked == NULL && PyErr_ExceptionMatches(struct_error)) { diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 916c65721e49258..7354082ea0490e6 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9660,6 +9660,7 @@ os_getlogin_impl(PyObject *module) errno = old_errno; } else { + _Py_MSAN_UNPOISON(name, sizeof(name)); result = PyUnicode_DecodeFSDefault(name); } #else @@ -16823,6 +16824,8 @@ os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) goto error; } + _Py_MSAN_UNPOISON(PyBytes_AS_STRING(bytes), n); + if (n != size) { _PyBytes_Resize(&bytes, n); } diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 6bb57b4da2229c4..b9dadaf6a865ffc 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -752,7 +752,9 @@ set_herror(socket_state *state, int h_error) PyObject *v; #ifdef HAVE_HSTRERROR - v = Py_BuildValue("(iN)", h_error, decode_error_message(hstrerror(h_error))); + const char *errmsg = hstrerror(h_error); + _Py_MSAN_UNPOISON_STRING(errmsg); + v = Py_BuildValue("(iN)", h_error, decode_error_message(errmsg)); #else v = Py_BuildValue("(is)", h_error, "host not found"); #endif @@ -779,7 +781,9 @@ set_gaierror(socket_state *state, int error) #endif #ifdef HAVE_GAI_STRERROR - v = Py_BuildValue("(iN)", error, decode_error_message(gai_strerror(error))); + const char *errmsg = gai_strerror(error); + _Py_MSAN_UNPOISON_STRING(errmsg); + v = Py_BuildValue("(iN)", error, decode_error_message(errmsg)); #else v = Py_BuildValue("(is)", error, "getaddrinfo failed"); #endif @@ -6408,6 +6412,7 @@ socket_getservbyport(PyObject *self, PyObject *args) PyErr_SetString(PyExc_OSError, "port/proto not found"); return NULL; } + _Py_MSAN_UNPOISON_STRING(sp->s_name); return PyUnicode_FromString(sp->s_name); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 0a7400c26993b32..22ca212a91bf5a7 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1661,6 +1661,7 @@ allocate_instrumentation_data(PyCodeObject *code) } monitoring->local_monitors = (_Py_LocalMonitors){ 0 }; monitoring->active_monitors = (_Py_LocalMonitors){ 0 }; + memset(monitoring->tool_versions, 0, sizeof(monitoring->tool_versions)); monitoring->tools = NULL; monitoring->lines = NULL; monitoring->line_tools = NULL; diff --git a/configure b/configure index 88f235951b6551c..9b0c81ac6f96a77 100755 --- a/configure +++ b/configure @@ -15892,7 +15892,7 @@ int main(void) { return 2; } - ffi_arg rc; + ffi_arg rc = 0; ffi_call(&cif, FFI_FN(z_is_expected), &rc, values); return !rc; } diff --git a/configure.ac b/configure.ac index a082848c50ca373..645ecfccc4846da 100644 --- a/configure.ac +++ b/configure.ac @@ -4247,7 +4247,7 @@ int main(void) { return 2; } - ffi_arg rc; + ffi_arg rc = 0; ffi_call(&cif, FFI_FN(z_is_expected), &rc, values); return !rc; } _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
