https://github.com/python/cpython/commit/44052b5f18c5d605d33bf3207b5c918127cf0e82
commit: 44052b5f18c5d605d33bf3207b5c918127cf0e82
branch: main
author: Nate Ohlson <nohl...@purdue.edu>
committer: hugovk <1324225+hug...@users.noreply.github.com>
date: 2024-09-16T20:22:30+03:00
summary:
gh-124064: Make warning emitting compiler options opt-in (#124070)
Co-authored-by: Hugo van Kemenade <1324225+hug...@users.noreply.github.com>
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
files:
A Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst
M .github/workflows/reusable-macos.yml
M .github/workflows/reusable-ubuntu.yml
M Tools/build/.warningignore_macos
M Tools/build/.warningignore_ubuntu
M Tools/build/check_warnings.py
M configure
M configure.ac
diff --git a/.github/workflows/reusable-macos.yml
b/.github/workflows/reusable-macos.yml
index d77723ef27c2dc..b4227545887ad1 100644
--- a/.github/workflows/reusable-macos.yml
+++ b/.github/workflows/reusable-macos.yml
@@ -35,7 +35,7 @@ jobs:
path: config.cache
key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{
inputs.config_hash }}
- name: Install Homebrew dependencies
- run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk
+ run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk make
- name: Configure CPython
run: |
GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
@@ -44,14 +44,27 @@ jobs:
--config-cache \
--with-pydebug \
--enable-slower-safety \
+ --enable-safety \
${{ inputs.free-threading && '--disable-gil' || '' }} \
--prefix=/opt/python-dev \
--with-openssl="$(brew --prefix openssl@3.0)"
- name: Build CPython
- run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt
+ if : ${{ inputs.free-threading || inputs.os != 'macos-13' }}
+ run: gmake -j8
+ - name: Build CPython for compiler warning check
+ if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+ run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee
compiler_output_macos.txt
- name: Display build info
run: make pythoninfo
- name: Check compiler warnings
- run: python3 Tools/build/check_warnings.py
--compiler-output-file-path=compiler_output.txt
--warning-ignore-file-path=Tools/build/.warningignore_macos
--compiler-output-type=clang
+ if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+ run: >-
+ python3 Tools/build/check_warnings.py
+ --compiler-output-file-path=compiler_output_macos.txt
+ --warning-ignore-file-path=Tools/build/.warningignore_macos
+ --compiler-output-type=clang
+ --fail-on-regression
+ --fail-on-improvement
+ --path-prefix="./"
- name: Tests
run: make test
diff --git a/.github/workflows/reusable-ubuntu.yml
b/.github/workflows/reusable-ubuntu.yml
index b197db814b2743..01bd914af79fa0 100644
--- a/.github/workflows/reusable-ubuntu.yml
+++ b/.github/workflows/reusable-ubuntu.yml
@@ -67,20 +67,33 @@ jobs:
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: >-
../cpython-ro-srcdir/configure
- CFLAGS="-fdiagnostics-format=json"
--config-cache
--with-pydebug
--enable-slower-safety
+ --enable-safety
--with-openssl=$OPENSSL_DIR
${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
- name: Build CPython out-of-tree
+ if: ${{ inputs.free-threading }}
working-directory: ${{ env.CPYTHON_BUILDDIR }}
- run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt
+ run: make -j4
+ - name: Build CPython out-of-tree (for compiler warning check)
+ if: ${{ !inputs.free-threading}}
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
+ run: set -o pipefail; make -j4 --output-sync 2>&1 | tee
compiler_output_ubuntu.txt
- name: Display build info
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make pythoninfo
- name: Check compiler warnings
- run: python Tools/build/check_warnings.py
--compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt
--warning-ignore-file-path
${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu
--compiler-output-type=json
+ if: ${{ !inputs.free-threading }}
+ run: >-
+ python Tools/build/check_warnings.py
+ --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR
}}/compiler_output_ubuntu.txt
+ --warning-ignore-file-path
${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu
+ --compiler-output-type=gcc
+ --fail-on-regression
+ --fail-on-improvement
+ --path-prefix="../cpython-ro-srcdir/"
- name: Remount sources writable for tests
# some tests write to srcdir, lack of pyc files slows down testing
run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
diff --git
a/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst
b/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst
new file mode 100644
index 00000000000000..9750cf203eef86
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst
@@ -0,0 +1 @@
+Enable compiler options that warn of potential security vulnerabilities.
diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos
index 67f50119db7310..3034638595353a 100644
--- a/Tools/build/.warningignore_macos
+++ b/Tools/build/.warningignore_macos
@@ -3,3 +3,230 @@
# Keep lines sorted lexicographically to help avoid merge conflicts.
# Format example:
# /path/to/file (number of warnings in file)
+Include/internal/mimalloc/mimalloc/internal.h 4
+Include/internal/pycore_backoff.h 1
+Include/internal/pycore_dict.h 2
+Include/internal/pycore_gc.h 1
+Include/internal/pycore_long.h 2
+Include/internal/pycore_object.h 4
+Modules/_asynciomodule.c 3
+Modules/_bisectmodule.c 2
+Modules/_bz2module.c 5
+Modules/_collectionsmodule.c 2
+Modules/_csv.c 3
+Modules/_ctypes/_ctypes.c 37
+Modules/_ctypes/_ctypes_test_generated.c.h 141
+Modules/_ctypes/callbacks.c 6
+Modules/_ctypes/callproc.c 15
+Modules/_ctypes/cfield.c 56
+Modules/_ctypes/malloc_closure.c 3
+Modules/_ctypes/stgdict.c 17
+Modules/_cursesmodule.c 24
+Modules/_datetimemodule.c 28
+Modules/_dbmmodule.c 8
+Modules/_decimal/_decimal.c 15
+Modules/_elementtree.c 42
+Modules/_functoolsmodule.c 6
+Modules/_gdbmmodule.c 5
+Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84
+Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84
+Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 24
+Modules/_hashopenssl.c 16
+Modules/_interpchannelsmodule.c 1
+Modules/_interpqueuesmodule.c 1
+Modules/_io/_iomodule.c 1
+Modules/_io/bufferedio.c 4
+Modules/_io/bytesio.c 11
+Modules/_io/fileio.c 9
+Modules/_io/stringio.c 8
+Modules/_io/textio.c 11
+Modules/_json.c 19
+Modules/_localemodule.c 3
+Modules/_lzmamodule.c 10
+Modules/_multiprocessing/semaphore.c 2
+Modules/_operator.c 5
+Modules/_pickle.c 71
+Modules/_posixsubprocess.c 8
+Modules/_queuemodule.c 4
+Modules/_randommodule.c 3
+Modules/_scproxy.c 3
+Modules/_sqlite/connection.c 4
+Modules/_sqlite/cursor.c 3
+Modules/_sqlite/module.c 2
+Modules/_sre/sre.c 18
+Modules/_sre/sre_lib.h 62
+Modules/_ssl.c 29
+Modules/_struct.c 1
+Modules/_testbuffer.c 22
+Modules/_testcapi/heaptype.c 1
+Modules/_testcapi/long.c 2
+Modules/_testcapi/mem.c 2
+Modules/_testcapi/monitoring.c 3
+Modules/_testcapi/pyatomic.c 1
+Modules/_testcapi/unicode.c 2
+Modules/_testcapi/vectorcall.c 3
+Modules/_testcapi/watchers.c 3
+Modules/_testcapimodule.c 3
+Modules/_testclinic.c 14
+Modules/_testexternalinspection.c 8
+Modules/_testinternalcapi.c 8
+Modules/_testinternalcapi/pytime.c 8
+Modules/_testinternalcapi/test_critical_sections.c 1
+Modules/_testinternalcapi/test_lock.c 2
+Modules/_testlimitedcapi/heaptype_relative.c 4
+Modules/_testlimitedcapi/object.c 2
+Modules/_testlimitedcapi/unicode.c 2
+Modules/_threadmodule.c 2
+Modules/_tkinter.c 6
+Modules/_xxtestfuzz/_xxtestfuzz.c 1
+Modules/_xxtestfuzz/fuzzer.c 11
+Modules/_zoneinfo.c 14
+Modules/arraymodule.c 32
+Modules/atexitmodule.c 1
+Modules/binascii.c 206
+Modules/blake2module.c 6
+Modules/cjkcodecs/_codecs_cn.c 1
+Modules/cjkcodecs/_codecs_iso2022.c 2
+Modules/cjkcodecs/_codecs_jp.c 14
+Modules/cjkcodecs/_codecs_kr.c 3
+Modules/cjkcodecs/cjkcodecs.h 1
+Modules/cjkcodecs/multibytecodec.c 2
+Modules/clinic/_testclinic.c.h 1
+Modules/clinic/arraymodule.c.h 1
+Modules/clinic/unicodedata.c.h 10
+Modules/cmathmodule.c 1
+Modules/expat/siphash.h 8
+Modules/expat/xmlparse.c 45
+Modules/expat/xmltok.c 17
+Modules/expat/xmltok_impl.c 34
+Modules/faulthandler.c 3
+Modules/fcntlmodule.c 1
+Modules/getpath.c 7
+Modules/grpmodule.c 4
+Modules/itertoolsmodule.c 7
+Modules/main.c 2
+Modules/mathmodule.c 15
+Modules/mmapmodule.c 20
+Modules/posixmodule.c 67
+Modules/pwdmodule.c 4
+Modules/pyexpat.c 20
+Modules/readline.c 1
+Modules/resource.c 3
+Modules/rotatingtree.c 1
+Modules/selectmodule.c 6
+Modules/sha3module.c 4
+Modules/signalmodule.c 1
+Modules/socketmodule.c 44
+Modules/syslogmodule.c 3
+Modules/timemodule.c 4
+Modules/unicodedata.c 28
+Modules/unicodedata_db.h 1
+Modules/xxsubtype.c 2
+Modules/zlibmodule.c 16
+Objects/abstract.c 2
+Objects/bytearrayobject.c 34
+Objects/bytes_methods.c 9
+Objects/bytesobject.c 35
+Objects/call.c 13
+Objects/classobject.c 4
+Objects/codeobject.c 15
+Objects/descrobject.c 2
+Objects/dictobject.c 28
+Objects/fileobject.c 3
+Objects/floatobject.c 30
+Objects/frameobject.c 19
+Objects/funcobject.c 1
+Objects/genobject.c 5
+Objects/listobject.c 43
+Objects/longobject.c 46
+Objects/memoryobject.c 6
+Objects/methodobject.c 1
+Objects/mimalloc/alloc.c 6
+Objects/mimalloc/arena.c 6
+Objects/mimalloc/heap.c 1
+Objects/mimalloc/init.c 2
+Objects/mimalloc/options.c 1
+Objects/mimalloc/os.c 4
+Objects/mimalloc/page-queue.c 2
+Objects/mimalloc/page.c 1
+Objects/mimalloc/prim/osx/../unix/prim.c 2
+Objects/mimalloc/random.c 1
+Objects/mimalloc/segment.c 11
+Objects/mimalloc/stats.c 1
+Objects/moduleobject.c 2
+Objects/object.c 1
+Objects/obmalloc.c 6
+Objects/odictobject.c 3
+Objects/rangeobject.c 10
+Objects/setobject.c 13
+Objects/sliceobject.c 4
+Objects/stringlib/codecs.h 26
+Objects/stringlib/eq.h 1
+Objects/stringlib/fastsearch.h 14
+Objects/stringlib/join.h 1
+Objects/stringlib/replace.h 4
+Objects/stringlib/repr.h 21
+Objects/stringlib/transmogrify.h 5
+Objects/structseq.c 14
+Objects/tupleobject.c 10
+Objects/typeobject.c 17
+Objects/unicodectype.c 7
+Objects/unicodeobject.c 113
+Parser/action_helpers.c 4
+Parser/lexer/buffer.c 1
+Parser/lexer/lexer.c 12
+Parser/parser.c 116
+Parser/pegen.c 7
+Parser/string_parser.c 7
+Parser/tokenizer/file_tokenizer.c 8
+Parser/tokenizer/helpers.c 7
+Parser/tokenizer/readline_tokenizer.c 3
+Programs/_freeze_module.c 1
+Python/Python-ast.c 15
+Python/asdl.c 3
+Python/assemble.c 7
+Python/ast_opt.c 7
+Python/bltinmodule.c 9
+Python/bootstrap_hash.c 4
+Python/ceval.c 8
+Python/ceval_gil.c 2
+Python/codecs.c 32
+Python/codegen.c 6
+Python/compile.c 2
+Python/context.c 1
+Python/crossinterp.c 2
+Python/crossinterp_data_lookup.h 1
+Python/dtoa.c 34
+Python/errors.c 1
+Python/fileutils.c 7
+Python/flowgraph.c 8
+Python/formatter_unicode.c 7
+Python/frame.c 4
+Python/gc.c 8
+Python/generated_cases.c.h 35
+Python/getargs.c 11
+Python/import.c 5
+Python/initconfig.c 11
+Python/instrumentation.c 31
+Python/intrinsics.c 1
+Python/legacy_tracing.c 3
+Python/lock.c 4
+Python/marshal.c 11
+Python/modsupport.c 3
+Python/mystrtoul.c 4
+Python/pathconfig.c 1
+Python/preconfig.c 2
+Python/pyarena.c 1
+Python/pyhash.c 2
+Python/pylifecycle.c 7
+Python/pystate.c 6
+Python/pystrhex.c 19
+Python/pystrtod.c 3
+Python/qsbr.c 2
+Python/specialize.c 10
+Python/suggestions.c 12
+Python/symtable.c 18
+Python/sysmodule.c 2
+Python/thread_pthread.h 1
+Python/traceback.c 6
+Python/tracemalloc.c 6
diff --git a/Tools/build/.warningignore_ubuntu
b/Tools/build/.warningignore_ubuntu
index 469c727abfb11c..e98305e81808d6 100644
--- a/Tools/build/.warningignore_ubuntu
+++ b/Tools/build/.warningignore_ubuntu
@@ -3,3 +3,263 @@
# Keep lines sorted lexicographically to help avoid merge conflicts.
# Format example:
# /path/to/file (number of warnings in file)
+/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/evp.h
2
+/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/ssl.h
4
+/usr/include/tcl8.6/tclTomMathDecls.h 1
+Include/cpython/bytearrayobject.h 1
+Include/cpython/bytesobject.h 3
+Include/cpython/dictobject.h 2
+Include/cpython/listobject.h 1
+Include/cpython/pyctype.h 2
+Include/cpython/tupleobject.h 1
+Include/cpython/unicodeobject.h 7
+Include/internal/mimalloc/mimalloc/internal.h 4
+Include/internal/mimalloc/mimalloc/types.h 2
+Include/internal/pycore_asdl.h 1
+Include/internal/pycore_backoff.h 3
+Include/internal/pycore_blocks_output_buffer.h 1
+Include/internal/pycore_dict.h 2
+Include/internal/pycore_gc.h 1
+Include/internal/pycore_gc.h 1
+Include/internal/pycore_interp.h 1
+Include/internal/pycore_list.h 1
+Include/internal/pycore_long.h 3
+Include/internal/pycore_object.h 4
+Include/internal/pycore_obmalloc.h 1
+Include/internal/pycore_pymath.h 1
+Include/internal/pycore_runtime_init.h 1
+Include/longobject.h 1
+Include/object.h 4
+Include/opcode_ids.h 1
+Include/pymacro.h 4
+Include/pymath.h 1
+Include/pymem.h 2
+Include/pyport.h 2
+Modules/_asynciomodule.c 3
+Modules/_bisectmodule.c 4
+Modules/_bz2module.c 5
+Modules/_collectionsmodule.c 2
+Modules/_csv.c 2
+Modules/_ctypes/_ctypes.c 53
+Modules/_ctypes/_ctypes_test.c 7
+Modules/_ctypes/_ctypes_test_generated.c.h 2
+Modules/_ctypes/callbacks.c 3
+Modules/_ctypes/callproc.c 13
+Modules/_ctypes/cfield.c 33
+Modules/_ctypes/stgdict.c 17
+Modules/_cursesmodule.c 27
+Modules/_datetimemodule.c 38
+Modules/_datetimemodule.c 38
+Modules/_dbmmodule.c 7
+Modules/_decimal/_decimal.c 19
+Modules/_elementtree.c 37
+Modules/_functoolsmodule.c 6
+Modules/_gdbmmodule.c 4
+Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84
+Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84
+Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84
+Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84
+Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4
+Modules/_hashopenssl.c 13
+Modules/_io/_iomodule.c 1
+Modules/_io/bufferedio.c 15
+Modules/_io/bytesio.c 14
+Modules/_io/fileio.c 9
+Modules/_io/stringio.c 8
+Modules/_io/textio.c 17
+Modules/_json.c 19
+Modules/_localemodule.c 2
+Modules/_lsprof.c 5
+Modules/_lzmamodule.c 6
+Modules/_multiprocessing/posixshmem.c 1
+Modules/_multiprocessing/semaphore.c 1
+Modules/_operator.c 5
+Modules/_pickle.c 73
+Modules/_posixsubprocess.c 11
+Modules/_queuemodule.c 4
+Modules/_randommodule.c 3
+Modules/_sqlite/connection.c 5
+Modules/_sqlite/cursor.c 3
+Modules/_sqlite/module.c 2
+Modules/_sre/sre.c 14
+Modules/_sre/sre_lib.h 25
+Modules/_ssl.c 26
+Modules/_struct.c 3
+Modules/_testbuffer.c 27
+Modules/_testcapi/bytes.c 1
+Modules/_testcapi/heaptype.c 1
+Modules/_testcapi/long.c 2
+Modules/_testcapi/mem.c 2
+Modules/_testcapi/monitoring.c 3
+Modules/_testcapi/pyatomic.c 4
+Modules/_testcapi/pyatomic.c 4
+Modules/_testcapi/unicode.c 1
+Modules/_testcapi/vectorcall.c 3
+Modules/_testcapi/watchers.c 3
+Modules/_testcapimodule.c 1
+Modules/_testclinic.c 14
+Modules/_testclinic.c 14
+Modules/_testexternalinspection.c 7
+Modules/_testinternalcapi.c 10
+Modules/_testinternalcapi/test_critical_sections.c 1
+Modules/_testinternalcapi/test_lock.c 4
+Modules/_testlimitedcapi/heaptype_relative.c 3
+Modules/_testlimitedcapi/object.c 2
+Modules/_testlimitedcapi/unicode.c 1
+Modules/_testmultiphase.c 1
+Modules/_tkinter.c 8
+Modules/_xxtestfuzz/_xxtestfuzz.c 1
+Modules/_xxtestfuzz/fuzzer.c 13
+Modules/_zoneinfo.c 17
+Modules/arraymodule.c 48
+Modules/binascii.c 208
+Modules/blake2module.c 8
+Modules/cjkcodecs/_codecs_iso2022.c 1
+Modules/cjkcodecs/_codecs_jp.c 17
+Modules/cjkcodecs/_codecs_kr.c 7
+Modules/cjkcodecs/alg_jisx0201.h 2
+Modules/cjkcodecs/cjkcodecs.h 1
+Modules/cjkcodecs/multibytecodec.c 12
+Modules/expat/pyexpatns.h 3
+Modules/expat/siphash.h 1
+Modules/expat/xmlparse.c 43
+Modules/expat/xmltok.c 15
+Modules/expat/xmltok.c 15
+Modules/expat/xmltok_impl.c 8
+Modules/faulthandler.c 5
+Modules/fcntlmodule.c 6
+Modules/getpath.c 7
+Modules/grpmodule.c 4
+Modules/itertoolsmodule.c 4
+Modules/main.c 2
+Modules/mathmodule.c 14
+Modules/mmapmodule.c 22
+Modules/mmapmodule.c 22
+Modules/posixmodule.c 79
+Modules/pwdmodule.c 4
+Modules/pyexpat.c 10
+Modules/readline.c 1
+Modules/resource.c 4
+Modules/rotatingtree.c 2
+Modules/selectmodule.c 1
+Modules/sha3module.c 4
+Modules/signalmodule.c 3
+Modules/socketmodule.c 75
+Modules/syslogmodule.c 3
+Modules/termios.c 1
+Modules/timemodule.c 10
+Modules/unicodedata.c 24
+Modules/unicodedata_db.h 1
+Modules/zlibmodule.c 24
+Objects/abstract.c 6
+Objects/bytearrayobject.c 42
+Objects/bytes_methods.c 4
+Objects/bytesobject.c 45
+Objects/call.c 12
+Objects/classobject.c 4
+Objects/codeobject.c 19
+Objects/descrobject.c 2
+Objects/dictobject.c 31
+Objects/fileobject.c 3
+Objects/floatobject.c 10
+Objects/frameobject.c 16
+Objects/funcobject.c 1
+Objects/genobject.c 3
+Objects/listobject.c 38
+Objects/longobject.c 47
+Objects/memoryobject.c 12
+Objects/methodobject.c 1
+Objects/mimalloc/alloc.c 6
+Objects/mimalloc/arena.c 6
+Objects/mimalloc/heap.c 2
+Objects/mimalloc/init.c 2
+Objects/mimalloc/options.c 4
+Objects/mimalloc/os.c 4
+Objects/mimalloc/page-queue.c 2
+Objects/mimalloc/page.c 2
+Objects/mimalloc/prim/unix/prim.c 6
+Objects/mimalloc/random.c 1
+Objects/mimalloc/segment.c 11
+Objects/mimalloc/stats.c 5
+Objects/moduleobject.c 4
+Objects/object.c 1
+Objects/obmalloc.c 6
+Objects/odictobject.c 6
+Objects/rangeobject.c 10
+Objects/setobject.c 13
+Objects/sliceobject.c 2
+Objects/stringlib/codecs.h 12
+Objects/stringlib/eq.h 1
+Objects/stringlib/fastsearch.h 8
+Objects/stringlib/join.h 3
+Objects/stringlib/replace.h 4
+Objects/stringlib/repr.h 21
+Objects/stringlib/transmogrify.h 26
+Objects/structseq.c 10
+Objects/tupleobject.c 8
+Objects/typeobject.c 38
+Objects/unicodectype.c 7
+Objects/unicodeobject.c 135
+Parser/action_helpers.c 3
+Parser/lexer/buffer.c 1
+Parser/lexer/lexer.c 14
+Parser/parser.c 116
+Parser/pegen.c 8
+Parser/string_parser.c 7
+Parser/tokenizer/file_tokenizer.c 9
+Parser/tokenizer/helpers.c 7
+Parser/tokenizer/readline_tokenizer.c 4
+Python/assemble.c 11
+Python/ast_opt.c 5
+Python/bltinmodule.c 8
+Python/bootstrap_hash.c 7
+Python/ceval.c 8
+Python/ceval_gil.c 2
+Python/codecs.c 28
+Python/codegen.c 6
+Python/compile.c 2
+Python/context.c 1
+Python/crossinterp.c 2
+Python/crossinterp_data_lookup.h 1
+Python/dtoa.c 30
+Python/errors.c 1
+Python/fileutils.c 11
+Python/flowgraph.c 7
+Python/formatter_unicode.c 6
+Python/frame.c 3
+Python/gc.c 9
+Python/gc.c 9
+Python/generated_cases.c.h 27
+Python/generated_cases.c.h 27
+Python/getargs.c 7
+Python/hashtable.c 1
+Python/import.c 6
+Python/import.c 7
+Python/initconfig.c 11
+Python/instrumentation.c 43
+Python/intrinsics.c 1
+Python/legacy_tracing.c 3
+Python/lock.c 4
+Python/marshal.c 16
+Python/modsupport.c 3
+Python/mystrtoul.c 4
+Python/pathconfig.c 1
+Python/perf_jit_trampoline.c 32
+Python/perf_trampoline.c 12
+Python/preconfig.c 2
+Python/pyarena.c 1
+Python/pyhash.c 4
+Python/pylifecycle.c 3
+Python/pystate.c 4
+Python/pystrhex.c 15
+Python/pystrtod.c 12
+Python/pytime.c 2
+Python/qsbr.c 2
+Python/specialize.c 9
+Python/suggestions.c 12
+Python/symtable.c 15
+Python/sysmodule.c 2
+Python/thread.c 1
+Python/thread_pthread.h 6
+Python/traceback.c 6
+Python/tracemalloc.c 6
diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py
index 1ed83447b6b668..e58ee2a7cc8571 100644
--- a/Tools/build/check_warnings.py
+++ b/Tools/build/check_warnings.py
@@ -1,105 +1,68 @@
"""
-Parses compiler output with -fdiagnostics-format=json and checks that warnings
+Parses compiler output from Clang or GCC and checks that warnings
exist only in files that are expected to have warnings.
"""
import argparse
from collections import defaultdict
-import json
import re
import sys
from pathlib import Path
from typing import NamedTuple
+
class FileWarnings(NamedTuple):
name: str
count: int
-def extract_warnings_from_compiler_output_clang(
+def extract_warnings_from_compiler_output(
compiler_output: str,
+ compiler_output_type: str,
+ path_prefix: str = "",
) -> list[dict]:
"""
- Extracts warnings from the compiler output when using clang
- """
- # Regex to find warnings in the compiler output
- clang_warning_regex = re.compile(
- r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: "
- r"(?P<message>.*) (?P<option>\[-[^\]]+\])$"
- )
- compiler_warnings = []
- for line in compiler_output.splitlines():
- if match := clang_warning_regex.match(line):
- compiler_warnings.append(
- {
- "file": match.group("file"),
- "line": match.group("line"),
- "column": match.group("column"),
- "message": match.group("message"),
- "option": match.group("option").lstrip("[").rstrip("]"),
- }
- )
-
- return compiler_warnings
-
-
-def extract_warnings_from_compiler_output_json(
- compiler_output: str,
-) -> list[dict]:
- """
- Extracts warnings from the compiler output when using
- -fdiagnostics-format=json.
-
- Compiler output as a whole is not a valid json document,
- but includes many json objects and may include other output
- that is not json.
+ Extracts warnings from the compiler output based on compiler
+ output type. Removes path prefix from file paths if provided.
+ Compatible with GCC and Clang compiler output.
"""
- # Regex to find json arrays at the top level of the file
- # in the compiler output
- json_arrays = re.findall(r"\[(?:[^[\]]|\[[^]]*])*]", compiler_output)
+ # Choose pattern and compile regex for particular compiler output
+ if compiler_output_type == "gcc":
+ regex_pattern = (
+ r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: "
+ r"(?P<message>.*?)(?: (?P<option>\[-[^\]]+\]))?$"
+ )
+ elif compiler_output_type == "clang":
+ regex_pattern = (
+ r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: "
+ r"(?P<message>.*) (?P<option>\[-[^\]]+\])$"
+ )
+ compiled_regex = re.compile(regex_pattern)
compiler_warnings = []
- for array in json_arrays:
- try:
- json_data = json.loads(array)
- json_objects_in_array = [entry for entry in json_data]
- warning_list = [
- entry
- for entry in json_objects_in_array
- if entry.get("kind") == "warning"
- ]
- for warning in warning_list:
- locations = warning["locations"]
- for location in locations:
- for key in ["caret", "start", "end"]:
- if key in location:
- compiler_warnings.append(
- {
- # Remove leading current directory if
present
- "file": location[key]["file"].lstrip("./"),
- "line": location[key]["line"],
- "column": location[key]["column"],
- "message": warning["message"],
- "option": warning["option"],
- }
- )
- # Found a caret, start, or end in location so
- # break out completely to address next warning
- break
- else:
- continue
- break
-
- except json.JSONDecodeError:
- continue # Skip malformed JSON
+ for i, line in enumerate(compiler_output.splitlines(), start=1):
+ if match := compiled_regex.match(line):
+ try:
+ compiler_warnings.append(
+ {
+ "file": match.group("file").removeprefix(path_prefix),
+ "line": match.group("line"),
+ "column": match.group("column"),
+ "message": match.group("message"),
+ "option":
match.group("option").lstrip("[").rstrip("]"),
+ }
+ )
+ except:
+ print(f"Error parsing compiler output. Unable to extract
warning on line {i}:\n{line}")
+ sys.exit(1)
return compiler_warnings
def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]:
"""
- Returns a dictionary where the key is the file and the data is the warnings
- in that file. Does not include duplicate warnings for a file from list of
- provided warnings.
+ Returns a dictionary where the key is the file and the data is the
+ warnings in that file. Does not include duplicate warnings for a
+ file from list of provided warnings.
"""
warnings_by_file = defaultdict(list)
warnings_added = set()
@@ -124,22 +87,28 @@ def get_unexpected_warnings(
are associated with a file that is not found in the list of files
with expected warnings
"""
- unexpected_warnings = []
+ unexpected_warnings = {}
for file in files_with_warnings.keys():
found_file_in_ignore_list = False
for ignore_file in files_with_expected_warnings:
if file == ignore_file.name:
if len(files_with_warnings[file]) > ignore_file.count:
- unexpected_warnings.extend(files_with_warnings[file])
+ unexpected_warnings[file] = (files_with_warnings[file],
ignore_file.count)
found_file_in_ignore_list = True
break
if not found_file_in_ignore_list:
- unexpected_warnings.extend(files_with_warnings[file])
+ unexpected_warnings[file] = (files_with_warnings[file], 0)
if unexpected_warnings:
print("Unexpected warnings:")
- for warning in unexpected_warnings:
- print(warning)
+ for file in unexpected_warnings:
+ print(
+ f"{file} expected {unexpected_warnings[file][1]} warnings,"
+ f" found {len(unexpected_warnings[file][0])}"
+ )
+ for warning in unexpected_warnings[file][0]:
+ print(warning)
+
return 1
return 0
@@ -156,14 +125,14 @@ def get_unexpected_improvements(
unexpected_improvements = []
for file in files_with_expected_warnings:
if file.name not in files_with_warnings.keys():
- unexpected_improvements.append(file)
+ unexpected_improvements.append((file.name, file.count, 0))
elif len(files_with_warnings[file.name]) < file.count:
- unexpected_improvements.append(file)
+ unexpected_improvements.append((file.name, file.count,
len(files_with_warnings[file.name])))
if unexpected_improvements:
print("Unexpected improvements:")
for file in unexpected_improvements:
- print(file.name)
+ print(f"{file[0]} expected {file[1]} warnings, found {file[2]}")
return 1
return 0
@@ -204,8 +173,15 @@ def main(argv: list[str] | None = None) -> int:
"--compiler-output-type",
type=str,
required=True,
- choices=["json", "clang"],
- help="Type of compiler output file (json or clang)",
+ choices=["gcc", "clang"],
+ help="Type of compiler output file (GCC or Clang)",
+ )
+ parser.add_argument(
+ "-p",
+ "--path-prefix",
+ type=str,
+ help="Path prefix to remove from the start of file paths"
+ " in compiler output",
)
args = parser.parse_args(argv)
@@ -241,7 +217,9 @@ def main(argv: list[str] | None = None) -> int:
# where the first element is the file name and the second element
# is the number of warnings expected in that file
files_with_expected_warnings = {
- FileWarnings(file.strip().split()[0],
int(file.strip().split()[1]))
+ FileWarnings(
+ file.strip().split()[0], int(file.strip().split()[1])
+ )
for file in clean_files
if file.strip() and not file.startswith("#")
}
@@ -249,14 +227,11 @@ def main(argv: list[str] | None = None) -> int:
with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f:
compiler_output_file_contents = f.read()
- if args.compiler_output_type == "json":
- warnings = extract_warnings_from_compiler_output_json(
- compiler_output_file_contents
- )
- elif args.compiler_output_type == "clang":
- warnings = extract_warnings_from_compiler_output_clang(
- compiler_output_file_contents
- )
+ warnings = extract_warnings_from_compiler_output(
+ compiler_output_file_contents,
+ args.compiler_output_type,
+ args.path_prefix
+ )
files_with_warnings = get_warnings_by_file(warnings)
@@ -272,6 +247,11 @@ def main(argv: list[str] | None = None) -> int:
if args.fail_on_improvement:
exit_code |= status
+ print(
+ "For information about this tool and its configuration"
+ " visit https://devguide.python.org/development-tools/warnings/";
+ )
+
return exit_code
diff --git a/configure b/configure
index 69301dec854fb1..ba4fc9301c8600 100755
--- a/configure
+++ b/configure
@@ -1829,7 +1829,7 @@ Optional Features:
(default is no)
--enable-bolt enable usage of the llvm-bolt post-link optimizer
(default is no)
- --disable-safety disable usage of the security compiler options with
+ --enable-safety enable usage of the security compiler options with
no performance overhead
--enable-slower-safety enable usage of the security compiler options with
performance overhead
@@ -9671,25 +9671,25 @@ fi
# Enable flags that warn and protect for potential security vulnerabilities.
# These flags should be enabled by default for all builds.
-{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --disable-safety"
>&5
-printf %s "checking for --disable-safety... " >&6; }
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-safety" >&5
+printf %s "checking for --enable-safety... " >&6; }
# Check whether --enable-safety was given.
if test ${enable_safety+y}
then :
- enableval=$enable_safety; if test "x$enable_safety" = xyes
+ enableval=$enable_safety; if test "x$disable_safety" = xyes
then :
- disable_safety=no
+ enable_safety=no
else $as_nop
- disable_safety=yes
+ enable_safety=yes
fi
else $as_nop
- disable_safety=no
+ enable_safety=no
fi
-{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $disable_safety" >&5
-printf "%s\n" "$disable_safety" >&6; }
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_safety" >&5
+printf "%s\n" "$enable_safety" >&6; }
-if test "$disable_safety" = "no"
+if test "$enable_safety" = "yes"
then
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler
accepts -fstack-protector-strong" >&5
printf %s "checking whether C compiler accepts -fstack-protector-strong... "
>&6; }
@@ -9767,6 +9767,201 @@ then :
else $as_nop
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wtrampolines not
supported" >&5
printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;}
+fi
+
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler
accepts -Wconversion" >&5
+printf %s "checking whether C compiler accepts -Wconversion... " >&6; }
+if test ${ax_cv_check_cflags__Werror__Wconversion+y}
+then :
+ printf %s "(cached) " >&6
+else $as_nop
+
+ ax_check_save_flags=$CFLAGS
+ CFLAGS="$CFLAGS -Werror -Wconversion"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main (void)
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+ ax_cv_check_cflags__Werror__Wconversion=yes
+else $as_nop
+ ax_cv_check_cflags__Werror__Wconversion=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+ CFLAGS=$ax_check_save_flags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result:
$ax_cv_check_cflags__Werror__Wconversion" >&5
+printf "%s\n" "$ax_cv_check_cflags__Werror__Wconversion" >&6; }
+if test "x$ax_cv_check_cflags__Werror__Wconversion" = xyes
+then :
+ CFLAGS_NODIST="$CFLAGS_NODIST -Wconversion"
+else $as_nop
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wconversion not
supported" >&5
+printf "%s\n" "$as_me: WARNING: -Wconversion not supported" >&2;}
+fi
+
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler
accepts -Wimplicit-fallthrough" >&5
+printf %s "checking whether C compiler accepts -Wimplicit-fallthrough... "
>&6; }
+if test ${ax_cv_check_cflags__Werror__Wimplicit_fallthrough+y}
+then :
+ printf %s "(cached) " >&6
+else $as_nop
+
+ ax_check_save_flags=$CFLAGS
+ CFLAGS="$CFLAGS -Werror -Wimplicit-fallthrough"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main (void)
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+ ax_cv_check_cflags__Werror__Wimplicit_fallthrough=yes
+else $as_nop
+ ax_cv_check_cflags__Werror__Wimplicit_fallthrough=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+ CFLAGS=$ax_check_save_flags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result:
$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&5
+printf "%s\n" "$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&6; }
+if test "x$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" = xyes
+then :
+ CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"
+else $as_nop
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING:
-Wimplicit-fallthrough not supported" >&5
+printf "%s\n" "$as_me: WARNING: -Wimplicit-fallthrough not supported" >&2;}
+fi
+
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler
accepts -Werror=format-security" >&5
+printf %s "checking whether C compiler accepts -Werror=format-security... "
>&6; }
+if test ${ax_cv_check_cflags__Werror__Werror_format_security+y}
+then :
+ printf %s "(cached) " >&6
+else $as_nop
+
+ ax_check_save_flags=$CFLAGS
+ CFLAGS="$CFLAGS -Werror -Werror=format-security"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main (void)
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+ ax_cv_check_cflags__Werror__Werror_format_security=yes
+else $as_nop
+ ax_cv_check_cflags__Werror__Werror_format_security=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+ CFLAGS=$ax_check_save_flags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result:
$ax_cv_check_cflags__Werror__Werror_format_security" >&5
+printf "%s\n" "$ax_cv_check_cflags__Werror__Werror_format_security" >&6; }
+if test "x$ax_cv_check_cflags__Werror__Werror_format_security" = xyes
+then :
+ CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"
+else $as_nop
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING:
-Werror=format-security not supported" >&5
+printf "%s\n" "$as_me: WARNING: -Werror=format-security not supported" >&2;}
+fi
+
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler
accepts -Wbidi-chars=any" >&5
+printf %s "checking whether C compiler accepts -Wbidi-chars=any... " >&6; }
+if test ${ax_cv_check_cflags__Werror__Wbidi_chars_any+y}
+then :
+ printf %s "(cached) " >&6
+else $as_nop
+
+ ax_check_save_flags=$CFLAGS
+ CFLAGS="$CFLAGS -Werror -Wbidi-chars=any"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main (void)
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+ ax_cv_check_cflags__Werror__Wbidi_chars_any=yes
+else $as_nop
+ ax_cv_check_cflags__Werror__Wbidi_chars_any=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+ CFLAGS=$ax_check_save_flags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result:
$ax_cv_check_cflags__Werror__Wbidi_chars_any" >&5
+printf "%s\n" "$ax_cv_check_cflags__Werror__Wbidi_chars_any" >&6; }
+if test "x$ax_cv_check_cflags__Werror__Wbidi_chars_any" = xyes
+then :
+ CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any"
+else $as_nop
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wbidi-chars=any not
supported" >&5
+printf "%s\n" "$as_me: WARNING: -Wbidi-chars=any not supported" >&2;}
+fi
+
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler
accepts -Wall" >&5
+printf %s "checking whether C compiler accepts -Wall... " >&6; }
+if test ${ax_cv_check_cflags__Werror__Wall+y}
+then :
+ printf %s "(cached) " >&6
+else $as_nop
+
+ ax_check_save_flags=$CFLAGS
+ CFLAGS="$CFLAGS -Werror -Wall"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main (void)
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+ ax_cv_check_cflags__Werror__Wall=yes
+else $as_nop
+ ax_cv_check_cflags__Werror__Wall=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+ CFLAGS=$ax_check_save_flags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result:
$ax_cv_check_cflags__Werror__Wall" >&5
+printf "%s\n" "$ax_cv_check_cflags__Werror__Wall" >&6; }
+if test "x$ax_cv_check_cflags__Werror__Wall" = xyes
+then :
+ CFLAGS_NODIST="$CFLAGS_NODIST -Wall"
+else $as_nop
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wall not supported"
>&5
+printf "%s\n" "$as_me: WARNING: -Wall not supported" >&2;}
fi
fi
diff --git a/configure.ac b/configure.ac
index ab39f46deeb8e5..5fc7e0822f89a8 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2500,16 +2500,21 @@ AS_VAR_IF([with_strict_overflow], [yes],
# Enable flags that warn and protect for potential security vulnerabilities.
# These flags should be enabled by default for all builds.
-AC_MSG_CHECKING([for --disable-safety])
+AC_MSG_CHECKING([for --enable-safety])
AC_ARG_ENABLE([safety],
- [AS_HELP_STRING([--disable-safety], [disable usage of the security compiler
options with no performance overhead])],
- [AS_VAR_IF([enable_safety], [yes], [disable_safety=no],
[disable_safety=yes])], [disable_safety=no])
-AC_MSG_RESULT([$disable_safety])
+ [AS_HELP_STRING([--enable-safety], [enable usage of the security compiler
options with no performance overhead])],
+ [AS_VAR_IF([disable_safety], [yes], [enable_safety=no],
[enable_safety=yes])], [enable_safety=no])
+AC_MSG_RESULT([$enable_safety])
-if test "$disable_safety" = "no"
+if test "$enable_safety" = "yes"
then
AX_CHECK_COMPILE_FLAG([-fstack-protector-strong],
[CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong"],
[AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wtrampolines], [CFLAGS_NODIST="$CFLAGS_NODIST
-Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror])
+ AX_CHECK_COMPILE_FLAG([-Wconversion], [CFLAGS_NODIST="$CFLAGS_NODIST
-Wconversion"], [AC_MSG_WARN([-Wconversion not supported])], [-Werror])
+ AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],
[CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"],
[AC_MSG_WARN([-Wimplicit-fallthrough not supported])], [-Werror])
+ AX_CHECK_COMPILE_FLAG([-Werror=format-security],
[CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"],
[AC_MSG_WARN([-Werror=format-security not supported])], [-Werror])
+ AX_CHECK_COMPILE_FLAG([-Wbidi-chars=any], [CFLAGS_NODIST="$CFLAGS_NODIST
-Wbidi-chars=any"], [AC_MSG_WARN([-Wbidi-chars=any not supported])], [-Werror])
+ AX_CHECK_COMPILE_FLAG([-Wall], [CFLAGS_NODIST="$CFLAGS_NODIST -Wall"],
[AC_MSG_WARN([-Wall not supported])], [-Werror])
fi
AC_MSG_CHECKING([for --enable-slower-safety])
_______________________________________________
Python-checkins mailing list -- python-checkins@python.org
To unsubscribe send an email to python-checkins-le...@python.org
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: arch...@mail-archive.com
