https://github.com/python/cpython/commit/26c0248b54b6b2a5df51dd3da8c0ebb1b2958bc4 commit: 26c0248b54b6b2a5df51dd3da8c0ebb1b2958bc4 branch: main author: Savannah Ostrowski <savannahostrow...@gmail.com> committer: savannahostrowski <savannahostrow...@gmail.com> date: 2025-04-30T11:03:57-07:00 summary:
GH-114809: Add support for macOS multi-arch builds with the JIT enabled (#131751) Co-authored-by: Ronald Oussoren <ronaldousso...@mac.com> Co-authored-by: Brandt Bucher <brandtbuc...@microsoft.com> files: A Misc/NEWS.d/next/Core_and_Builtins/2025-03-26-04-55-25.gh-issue-114809.8rNyT7.rst M .github/workflows/jit.yml M .gitignore M Tools/jit/_targets.py M Tools/jit/build.py M configure M configure.ac diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index e08d3c8bfd61a8..af460f4264932c 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -113,7 +113,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} + ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.gitignore b/.gitignore index 360934041c7d38..2a6f249275c32e 100644 --- a/.gitignore +++ b/.gitignore @@ -138,7 +138,7 @@ Tools/unicode/data/ # hendrikmuhs/ccache-action@v1 /.ccache /cross-build/ -/jit_stencils.h +/jit_stencils*.h /platform /profile-clean-stamp /profile-run-stamp diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-03-26-04-55-25.gh-issue-114809.8rNyT7.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-26-04-55-25.gh-issue-114809.8rNyT7.rst new file mode 100644 index 00000000000000..19d92c33bc6d21 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-03-26-04-55-25.gh-issue-114809.8rNyT7.rst @@ -0,0 +1 @@ +Add support for macOS multi-arch builds with the JIT enabled diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index f7ea159884c5a4..e0ac735a20f691 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -25,6 +25,7 @@ CPYTHON = TOOLS.parent PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h" TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" +ASYNCIO_RUNNER = asyncio.Runner() _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) _R = typing.TypeVar( @@ -35,6 +36,7 @@ @dataclasses.dataclass class _Target(typing.Generic[_S, _R]): triple: str + condition: str _: dataclasses.KW_ONLY alignment: int = 1 args: typing.Sequence[str] = () @@ -188,7 +190,12 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: return stencil_groups def build( - self, out: pathlib.Path, *, comment: str = "", force: bool = False + self, + out: pathlib.Path, + *, + comment: str = "", + force: bool = False, + stencils_h: str = "jit_stencils.h", ) -> None: """Build jit_stencils.h in the given directory.""" if not self.stable: @@ -197,14 +204,14 @@ def build( outline = "=" * len(warning) print("\n".join(["", outline, warning, request, outline, ""])) digest = f"// {self._compute_digest(out)}\n" - jit_stencils = out / "jit_stencils.h" + jit_stencils = out / stencils_h if ( not force and jit_stencils.exists() and jit_stencils.read_text().startswith(digest) ): return - stencil_groups = asyncio.run(self._build_stencils()) + stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) jit_stencils_new = out / "jit_stencils.h.new" try: with jit_stencils_new.open("w") as file: @@ -512,10 +519,12 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" target: _COFF | _ELF | _MachO if re.fullmatch(r"aarch64-apple-darwin.*", host): - target = _MachO(host, alignment=8, prefix="_") + condition = "defined(__aarch64__) && defined(__APPLE__)" + target = _MachO(host, condition, alignment=8, prefix="_") elif re.fullmatch(r"aarch64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll", "-fplt"] - target = _COFF(host, alignment=8, args=args) + condition = "defined(_M_ARM64)" + target = _COFF(host, condition, alignment=8, args=args) elif re.fullmatch(r"aarch64-.*-linux-gnu", host): args = [ "-fpic", @@ -523,22 +532,27 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: # was required to disable them. "-mno-outline-atomics", ] - target = _ELF(host, alignment=8, args=args) + condition = "defined(__aarch64__) && defined(__linux__)" + target = _ELF(host, condition, alignment=8, args=args) elif re.fullmatch(r"i686-pc-windows-msvc", host): args = [ "-DPy_NO_ENABLE_SHARED", # __attribute__((preserve_none)) is not supported "-Wno-ignored-attributes", ] - target = _COFF(host, args=args, prefix="_") + condition = "defined(_M_IX86)" + target = _COFF(host, condition, args=args, prefix="_") elif re.fullmatch(r"x86_64-apple-darwin.*", host): - target = _MachO(host, prefix="_") + condition = "defined(__x86_64__) && defined(__APPLE__)" + target = _MachO(host, condition, prefix="_") elif re.fullmatch(r"x86_64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] - target = _COFF(host, args=args) + condition = "defined(_M_X64)" + target = _COFF(host, condition, args=args) elif re.fullmatch(r"x86_64-.*-linux-gnu", host): args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0"] - target = _ELF(host, args=args) + condition = "defined(__x86_64__) && defined(__linux__)" + target = _ELF(host, condition, args=args) else: raise ValueError(host) return target diff --git a/Tools/jit/build.py b/Tools/jit/build.py index a8cb0f67c36363..4d1e484b6838eb 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -11,7 +11,10 @@ comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "target", type=_targets.get_target, help="a PEP 11 target triple to compile for" + "target", + nargs="+", + type=_targets.get_target, + help="a PEP 11 target triple to compile for", ) parser.add_argument( "-d", "--debug", action="store_true", help="compile for a debug build of Python" @@ -23,6 +26,22 @@ "-v", "--verbose", action="store_true", help="echo commands as they are run" ) args = parser.parse_args() - args.target.debug = args.debug - args.target.verbose = args.verbose - args.target.build(pathlib.Path.cwd(), comment=comment, force=args.force) + for target in args.target: + target.debug = args.debug + target.force = args.force + target.verbose = args.verbose + target.build( + pathlib.Path.cwd(), + comment=comment, + stencils_h=f"jit_stencils-{target.triple}.h", + force=args.force, + ) + + with open("jit_stencils.h", "w") as fp: + for idx, target in enumerate(args.target): + fp.write(f"#{'if' if idx == 0 else 'elif'} {target.condition}\n") + fp.write(f'#include "jit_stencils-{target.triple}.h"\n') + + fp.write("#else\n") + fp.write('#error "unexpected target"\n') + fp.write("#endif\n") diff --git a/configure b/configure index 58c4a646c934db..1ec91021a38b66 100755 --- a/configure +++ b/configure @@ -900,6 +900,8 @@ LDSHARED SHLIB_SUFFIX DSYMUTIL_PATH DSYMUTIL +JIT_STENCILS_H +REGEN_JIT_COMMAND UNIVERSAL_ARCH_FLAGS WASM_STDLIB WASM_ASSETS_DIR @@ -928,8 +930,6 @@ LLVM_AR PROFILE_TASK DEF_MAKE_RULE DEF_MAKE_ALL_RULE -JIT_STENCILS_H -REGEN_JIT_COMMAND ABI_THREAD ABIFLAGS LN @@ -1092,13 +1092,13 @@ with_pydebug with_trace_refs enable_pystats with_assertions -enable_experimental_jit enable_optimizations with_lto enable_bolt with_strict_overflow enable_safety enable_slower_safety +enable_experimental_jit with_dsymutil with_address_sanitizer with_memory_sanitizer @@ -1823,9 +1823,6 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) - --enable-experimental-jit[=no|yes|yes-off|interpreter] - build the experimental just-in-time compiler - (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer @@ -1834,6 +1831,9 @@ Optional Features: no performance overhead --enable-slower-safety enable usage of the security compiler options with performance overhead + --enable-experimental-jit[=no|yes|yes-off|interpreter] + build the experimental just-in-time compiler + (default is no) --enable-loadable-sqlite-extensions support loadable extensions in the sqlite3 module, see Doc/library/sqlite3.rst (default is no) @@ -8367,51 +8367,6 @@ else printf "%s\n" "no" >&6; } fi -# Check for --enable-experimental-jit: -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5 -printf %s "checking for --enable-experimental-jit... " >&6; } -# Check whether --enable-experimental-jit was given. -if test ${enable_experimental_jit+y} -then : - enableval=$enable_experimental_jit; -else case e in #( - e) enable_experimental_jit=no ;; -esac -fi - -case $enable_experimental_jit in - no) jit_flags=""; tier2_flags="" ;; - yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; - yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; - interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; - interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option - *) as_fn_error $? "invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter" "$LINENO" 5 ;; -esac -if ${tier2_flags:+false} : -then : - -else case e in #( - e) as_fn_append CFLAGS_NODIST " $tier2_flags" ;; -esac -fi -if ${jit_flags:+false} : -then : - -else case e in #( - e) as_fn_append CFLAGS_NODIST " $jit_flags" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" - JIT_STENCILS_H="jit_stencils.h" - if test "x$Py_DEBUG" = xtrue -then : - as_fn_append REGEN_JIT_COMMAND " --debug" -fi ;; -esac -fi - - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $tier2_flags $jit_flags" >&5 -printf "%s\n" "$tier2_flags $jit_flags" >&6; } - # Enable optimization flags @@ -10706,42 +10661,50 @@ fi UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPLES=`echo {ppc,i386}-apple-darwin` ;; 64-bit) UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPLES=`echo {ppc64,x86_64}-apple-darwin` ;; all) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin` ;; universal2) UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" LIPO_32BIT_FLAGS="" LIPO_INTEL64_FLAGS="-extract x86_64" ARCH_RUN_32BIT="true" + ARCH_TRIPLES=`echo {aarch64,x86_64}-apple-darwin` ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386" + ARCH_TRIPLES=`echo {i386,x86_64}-apple-darwin` ;; intel-32) UNIVERSAL_ARCH_FLAGS="-arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPLES=i386-apple-darwin ;; intel-64) UNIVERSAL_ARCH_FLAGS="-arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPLES=x86_64-apple-darwin ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPLES=`echo {i386,ppc,x86_64}-apple-darwin` ;; *) as_fn_error $? "proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way" "$LINENO" 5 @@ -10858,6 +10821,51 @@ else case e in #( esac fi +# Check for --enable-experimental-jit: +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-experimental-jit" >&5 +printf %s "checking for --enable-experimental-jit... " >&6; } +# Check whether --enable-experimental-jit was given. +if test ${enable_experimental_jit+y} +then : + enableval=$enable_experimental_jit; +else case e in #( + e) enable_experimental_jit=no ;; +esac +fi + +case $enable_experimental_jit in + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option + *) as_fn_error $? "invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter" "$LINENO" 5 ;; +esac +if ${tier2_flags:+false} : +then : + +else case e in #( + e) as_fn_append CFLAGS_NODIST " $tier2_flags" ;; +esac +fi +if ${jit_flags:+false} : +then : + +else case e in #( + e) as_fn_append CFLAGS_NODIST " $jit_flags" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}" + JIT_STENCILS_H="jit_stencils.h" + if test "x$Py_DEBUG" = xtrue +then : + as_fn_append REGEN_JIT_COMMAND " --debug" +fi ;; +esac +fi + + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $tier2_flags $jit_flags" >&5 +printf "%s\n" "$tier2_flags $jit_flags" >&6; } + case "$ac_cv_cc_name" in mpicc) CFLAGS_NODIST="$CFLAGS_NODIST" diff --git a/configure.ac b/configure.ac index ee2cabbf7cd268..16367b3ea53c27 100644 --- a/configure.ac +++ b/configure.ac @@ -1802,41 +1802,6 @@ else AC_MSG_RESULT([no]) fi -# Check for --enable-experimental-jit: -AC_MSG_CHECKING([for --enable-experimental-jit]) -AC_ARG_ENABLE([experimental-jit], - [AS_HELP_STRING([--enable-experimental-jit@<:@=no|yes|yes-off|interpreter@:>@], - [build the experimental just-in-time compiler (default is no)])], - [], - [enable_experimental_jit=no]) -case $enable_experimental_jit in - no) jit_flags=""; tier2_flags="" ;; - yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; - yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; - interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; - interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option - *) AC_MSG_ERROR( - [invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter]) ;; -esac -AS_VAR_IF([tier2_flags], - [], - [], - [AS_VAR_APPEND([CFLAGS_NODIST], [" $tier2_flags"])]) -AS_VAR_IF([jit_flags], - [], - [], - [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) - AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) - AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) - AS_VAR_IF([Py_DEBUG], - [true], - [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], - [])]) -AC_SUBST([REGEN_JIT_COMMAND]) -AC_SUBST([JIT_STENCILS_H]) -AC_MSG_RESULT([$tier2_flags $jit_flags]) - # Enable optimization flags AC_SUBST([DEF_MAKE_ALL_RULE]) AC_SUBST([DEF_MAKE_RULE]) @@ -2652,42 +2617,50 @@ AS_VAR_IF([ac_cv_gcc_compat], [yes], [ UNIVERSAL_ARCH_FLAGS="-arch ppc -arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPLES=`echo {ppc,i386}-apple-darwin` ;; 64-bit) UNIVERSAL_ARCH_FLAGS="-arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPLES=`echo {ppc64,x86_64}-apple-darwin` ;; all) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch ppc64 -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPLES=`echo {i386,ppc,ppc64,x86_64}-apple-darwin` ;; universal2) UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" LIPO_32BIT_FLAGS="" LIPO_INTEL64_FLAGS="-extract x86_64" ARCH_RUN_32BIT="true" + ARCH_TRIPLES=`echo {aarch64,x86_64}-apple-darwin` ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386" + ARCH_TRIPLES=`echo {i386,x86_64}-apple-darwin` ;; intel-32) UNIVERSAL_ARCH_FLAGS="-arch i386" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="" + ARCH_TRIPLES=i386-apple-darwin ;; intel-64) UNIVERSAL_ARCH_FLAGS="-arch x86_64" LIPO_32BIT_FLAGS="" ARCH_RUN_32BIT="true" + ARCH_TRIPLES=x86_64-apple-darwin ;; 3-way) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch ppc -arch x86_64" LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" + ARCH_TRIPLES=`echo {i386,ppc,x86_64}-apple-darwin` ;; *) AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way]) @@ -2778,6 +2751,41 @@ AS_VAR_IF([ac_cv_gcc_compat], [yes], [ esac ]) +# Check for --enable-experimental-jit: +AC_MSG_CHECKING([for --enable-experimental-jit]) +AC_ARG_ENABLE([experimental-jit], + [AS_HELP_STRING([--enable-experimental-jit@<:@=no|yes|yes-off|interpreter@:>@], + [build the experimental just-in-time compiler (default is no)])], + [], + [enable_experimental_jit=no]) +case $enable_experimental_jit in + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option + *) AC_MSG_ERROR( + [invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter]) ;; +esac +AS_VAR_IF([tier2_flags], + [], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" $tier2_flags"])]) +AS_VAR_IF([jit_flags], + [], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) + AS_VAR_SET([REGEN_JIT_COMMAND], + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}"]) + AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) + AS_VAR_IF([Py_DEBUG], + [true], + [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], + [])]) +AC_SUBST([REGEN_JIT_COMMAND]) +AC_SUBST([JIT_STENCILS_H]) +AC_MSG_RESULT([$tier2_flags $jit_flags]) + case "$ac_cv_cc_name" in mpicc) CFLAGS_NODIST="$CFLAGS_NODIST" _______________________________________________ Python-checkins mailing list -- python-checkins@python.org To unsubscribe send an email to python-checkins-le...@python.org https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: arch...@mail-archive.com