https://github.com/python/cpython/commit/7025d751e591427392e20cdb685df9c31e622893 commit: 7025d751e591427392e20cdb685df9c31e622893 branch: 3.13 author: Seth Michael Larson <[email protected]> committer: hugovk <[email protected]> date: 2026-01-16T19:24:34+02:00 summary:
[3.13] gh-143572: Run 'python3-libraries' fuzzer in CI using CIFuzz (… (#143915) Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) <[email protected]> Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) <[email protected]> Co-authored-by: Hugo van Kemenade <[email protected]> files: A .github/workflows/reusable-cifuzz.yml M .github/workflows/build.yml M .github/workflows/reusable-context.yml M Tools/build/compute-changes.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b496cd020cfa67..5246e6c39fcf29 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -543,45 +543,45 @@ jobs: sanitizer: ${{ matrix.sanitizer }} free-threading: ${{ matrix.free-threading }} - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: - name: CIFuzz - runs-on: ubuntu-latest - timeout-minutes: 60 + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: CIFuzz${{ '' }} # zizmor: ignore[obfuscation] needs: build-context - if: needs.build-context.outputs.run-ci-fuzz == 'true' + if: >- + needs.build-context.outputs.run-ci-fuzz == 'true' + || needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' permissions: security-events: write strategy: fail-fast: false matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 - oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v6 - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif + sanitizer: + - address + - undefined + - memory + oss-fuzz-project-name: + - cpython3 + - python3-libraries + exclude: + # Note that the 'no-exclude' sentinel below is to prevent + # an empty string value from excluding all jobs and causing + # GHA to create a 'default' matrix entry with all empty values. + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz == 'true' + && 'no-exclude' + || 'cpython3' + }} + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' + && 'no-exclude' + || 'python3-libraries' + }} + uses: ./.github/workflows/reusable-cifuzz.yml + with: + oss-fuzz-project-name: ${{ matrix.oss-fuzz-project-name }} + sanitizer: ${{ matrix.sanitizer }} all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass @@ -625,7 +625,12 @@ jobs: || '' }} ${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }} - ${{ !fromJSON(needs.build-context.outputs.run-ci-fuzz) && 'cifuzz,' || '' }} + ${{ + !fromJSON(needs.build-context.outputs.run-ci-fuzz) + && !fromJSON(needs.build-context.outputs.run-ci-fuzz-stdlib) + && 'cifuzz,' || + '' + }} ${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-ubuntu) diff --git a/.github/workflows/reusable-cifuzz.yml b/.github/workflows/reusable-cifuzz.yml new file mode 100644 index 00000000000000..1986f5fb2cc640 --- /dev/null +++ b/.github/workflows/reusable-cifuzz.yml @@ -0,0 +1,46 @@ +# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ +name: Reusable CIFuzz + +on: + workflow_call: + inputs: + oss-fuzz-project-name: + description: OSS-Fuzz project name + required: true + type: string + sanitizer: + description: OSS-Fuzz sanitizer + required: true + type: string + +jobs: + cifuzz: + name: ${{ inputs.oss-fuzz-project-name }} (${{ inputs.sanitizer }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Build fuzzers (${{ inputs.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + sanitizer: ${{ inputs.sanitizer }} + - name: Run fuzzers (${{ inputs.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + output-sarif: true + sanitizer: ${{ inputs.sanitizer }} + - name: Upload crash + if: failure() && steps.build.outcome == 'success' + uses: actions/upload-artifact@v6 + with: + name: ${{ inputs.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index 44923ef7b1f55b..ee4d811fdf7ea3 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -21,8 +21,11 @@ on: # yamllint disable-line rule:truthy description: Whether to run the Android tests value: ${{ jobs.compute-changes.outputs.run-android }} # bool run-ci-fuzz: - description: Whether to run the CIFuzz job + description: Whether to run the CIFuzz job for 'cpython' fuzzer value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + run-ci-fuzz-stdlib: + description: Whether to run the CIFuzz job for 'python3-libraries' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz-stdlib }} # bool run-docs: description: Whether to build the docs value: ${{ jobs.compute-changes.outputs.run-docs }} # bool @@ -53,6 +56,7 @@ jobs: outputs: run-android: ${{ steps.changes.outputs.run-android }} run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} + run-ci-fuzz-stdlib: ${{ steps.changes.outputs.run-ci-fuzz-stdlib }} run-docs: ${{ steps.changes.outputs.run-docs }} run-macos: ${{ steps.changes.outputs.run-macos }} run-tests: ${{ steps.changes.outputs.run-tests }} diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 7be2503c7dffda..485e0d621c37bd 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -11,7 +11,7 @@ import os import subprocess -from dataclasses import dataclass +from dataclasses import dataclass, fields from pathlib import Path TYPE_CHECKING = False @@ -51,11 +51,59 @@ MACOS_DIRS = frozenset({"Mac"}) WASI_DIRS = frozenset({Path("Tools", "wasm")}) +LIBRARY_FUZZER_PATHS = frozenset({ + # All C/CPP fuzzers. + Path("configure"), + Path(".github/workflows/reusable-cifuzz.yml"), + # ast + Path("Lib/ast.py"), + Path("Python/ast.c"), + # configparser + Path("Lib/configparser.py"), + # csv + Path("Lib/csv.py"), + Path("Modules/_csv.c"), + # decode + Path("Lib/encodings/"), + Path("Modules/_codecsmodule.c"), + Path("Modules/cjkcodecs/"), + Path("Modules/unicodedata*"), + # difflib + Path("Lib/difflib.py"), + # email + Path("Lib/email/"), + # html + Path("Lib/html/"), + Path("Lib/_markupbase.py"), + # http.client + Path("Lib/http/client.py"), + # json + Path("Lib/json/"), + Path("Modules/_json.c"), + # plist + Path("Lib/plistlib.py"), + # re + Path("Lib/re/"), + Path("Modules/_sre/"), + # tarfile + Path("Lib/tarfile.py"), + # tomllib + Path("Modules/tomllib/"), + # xml + Path("Lib/xml/"), + Path("Lib/_markupbase.py"), + Path("Modules/expat/"), + Path("Modules/pyexpat.c"), + # zipfile + Path("Lib/zipfile/"), +}) + @dataclass(kw_only=True, slots=True) class Outputs: run_android: bool = False run_ci_fuzz: bool = False + run_ci_fuzz_stdlib: bool = False run_docs: bool = False run_macos: bool = False run_tests: bool = False @@ -93,6 +141,11 @@ def compute_changes() -> None: else: print("Branch too old for CIFuzz tests; or no C files were changed") + if outputs.run_ci_fuzz_stdlib: + print("Run CIFuzz tests for stdlib") + else: + print("Branch too old for CIFuzz tests; or no stdlib files were changed") + if outputs.run_docs: print("Build documentation") @@ -141,9 +194,18 @@ def get_file_platform(file: Path) -> str | None: return None +def is_fuzzable_library_file(file: Path) -> bool: + return any( + (file.is_relative_to(needs_fuzz) and needs_fuzz.is_dir()) + or (file == needs_fuzz and file.is_file()) + for needs_fuzz in LIBRARY_FUZZER_PATHS + ) + + def process_changed_files(changed_files: Set[Path]) -> Outputs: run_tests = False run_ci_fuzz = False + run_ci_fuzz_stdlib = False run_docs = False run_windows_tests = False run_windows_msi = False @@ -157,8 +219,8 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: doc_file = file.suffix in SUFFIXES_DOCUMENTATION or doc_or_misc if file.parent == GITHUB_WORKFLOWS_PATH: - if file.name == "build.yml": - run_tests = run_ci_fuzz = True + if file.name in ("build.yml", "reusable-cifuzz.yml"): + run_tests = run_ci_fuzz = run_ci_fuzz_stdlib = True has_platform_specific_change = False if file.name == "reusable-docs.yml": run_docs = True @@ -189,6 +251,8 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: ("Modules", "_xxtestfuzz"), }: run_ci_fuzz = True + if not run_ci_fuzz_stdlib and is_fuzzable_library_file(file): + run_ci_fuzz_stdlib = True # Check for changed documentation-related files if doc_file: @@ -219,6 +283,7 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: return Outputs( run_android=run_android, run_ci_fuzz=run_ci_fuzz, + run_ci_fuzz_stdlib=run_ci_fuzz_stdlib, run_docs=run_docs, run_macos=run_macos, run_tests=run_tests, @@ -252,15 +317,10 @@ def write_github_output(outputs: Outputs) -> None: return with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: - f.write(f"run-android={bool_lower(outputs.run_android)}\n") - f.write(f"run-ci-fuzz={bool_lower(outputs.run_ci_fuzz)}\n") - f.write(f"run-docs={bool_lower(outputs.run_docs)}\n") - f.write(f"run-macos={bool_lower(outputs.run_macos)}\n") - f.write(f"run-tests={bool_lower(outputs.run_tests)}\n") - f.write(f"run-ubuntu={bool_lower(outputs.run_ubuntu)}\n") - f.write(f"run-wasi={bool_lower(outputs.run_wasi)}\n") - f.write(f"run-windows-msi={bool_lower(outputs.run_windows_msi)}\n") - f.write(f"run-windows-tests={bool_lower(outputs.run_windows_tests)}\n") + for field in fields(outputs): + name = field.name.replace("_", "-") + val = bool_lower(getattr(outputs, field.name)) + f.write(f"{name}={val}\n") def bool_lower(value: bool, /) -> str: _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
