https://github.com/python/cpython/commit/5a3f479601e52d694cc21415cb925037dffc1138 commit: 5a3f479601e52d694cc21415cb925037dffc1138 branch: main author: Uwe L. Korn <[email protected]> committer: savannahostrowski <[email protected]> date: 2026-04-20T16:45:53Z summary:
gh-138451: Support custom LLVM installation path (#138452) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Steve Dower <[email protected]> Co-authored-by: Savannah Ostrowski <[email protected]> files: A Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst M PCbuild/regen.targets M Tools/jit/README.md M Tools/jit/_llvm.py M Tools/jit/_targets.py M Tools/jit/build.py M configure M configure.ac diff --git a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst new file mode 100644 index 00000000000000..d83aee08025502 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst @@ -0,0 +1 @@ +Allow for custom LLVM path using ``LLVM_TOOLS_INSTALL_DIR`` during JIT build. diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 41af9cacfb912b..bb059f382eb375 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -129,7 +129,7 @@ <JITArgs Condition="$(Platform) == 'x64'">x86_64-pc-windows-msvc</JITArgs> <JITArgs Condition="$(Configuration) == 'Debug'">$(JITArgs) --debug</JITArgs> </PropertyGroup> - <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs) --output-dir "$(GeneratedJitStencilsDir)" --pyconfig-dir "$(PySourcePath)PC"'/> + <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs) --output-dir "$(GeneratedJitStencilsDir)" --pyconfig-dir "$(PySourcePath)PC" --llvm-version="$(LLVM_VERSION)" --llvm-tools-install-dir="$(LLVM_TOOLS_INSTALL_DIR)"'/> </Target> <Target Name="_CleanJIT" AfterTargets="Clean"> <Delete Files="@(_JITOutputs)"/> diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 8eadb3349ba6da..fd7154d0e76d0a 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,7 +9,12 @@ Python 3.11 or newer is required to build the JIT. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-21`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. + +You can customize the LLVM configuration using environment variables before running configure: + +- LLVM_VERSION: Specify a different LLVM version (default: 21) +- LLVM_TOOLS_INSTALL_DIR: Point to a specific LLVM installation prefix when multiple installations exist (the tools are expected in `<dir>/bin`) It's easy to install all of the required tools: @@ -62,7 +67,7 @@ choco install llvm --version=21.1.0 ### Dev Containers -If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no +If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index a4aaacdf41249d..601752bf1f6396 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -80,7 +80,18 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str @_async_cache -async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: +async def _find_tool( + tool: str, + llvm_version: str, + llvm_tools_install_dir: str | None, + *, + echo: bool = False, +) -> str | None: + # Explicitly defined LLVM installation location + if llvm_tools_install_dir: + path = os.path.join(llvm_tools_install_dir, "bin", tool) + if await _check_tool_version(path, llvm_version, echo=echo): + return path # Unversioned executables: path = tool if await _check_tool_version(path, llvm_version, echo=echo): @@ -114,10 +125,11 @@ async def maybe_run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str | None: """Run an LLVM tool if it can be found. Otherwise, return None.""" - path = await _find_tool(tool, llvm_version, echo=echo) + path = await _find_tool(tool, llvm_version, llvm_tools_install_dir, echo=echo) return path and await _run(path, args, echo=echo) @@ -126,10 +138,17 @@ async def run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str: """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" - output = await maybe_run(tool, args, echo=echo, llvm_version=llvm_version) + output = await maybe_run( + tool, + args, + echo=echo, + llvm_version=llvm_version, + llvm_tools_install_dir=llvm_tools_install_dir, + ) if output is None: raise RuntimeError(f"Can't find {tool}-{llvm_version}!") return output diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index fd5c143b8a812f..f78e80db165fc8 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -53,6 +53,7 @@ class _Target(typing.Generic[_S, _R]): cflags: str = "" frame_pointers: bool = False llvm_version: str = _llvm._LLVM_VERSION + llvm_tools_install_dir: str | None = None known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() @@ -85,7 +86,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: group = _stencils.StencilGroup() args = ["--disassemble", "--reloc", f"{path}"] output = await _llvm.maybe_run( - "llvm-objdump", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-objdump", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) if output is not None: # Make sure that full paths don't leak out (for reproducibility): @@ -105,7 +110,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: f"{path}", ] output = await _llvm.run( - "llvm-readobj", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-readobj", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) # --elf-output-style=JSON is only *slightly* broken on Mach-O... output = output.replace("PrivateExtern\n", "\n") @@ -184,7 +193,11 @@ async def _compile( # Allow user-provided CFLAGS to override any defaults args_s += shlex.split(self.cflags) await _llvm.run( - "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_s, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) if not is_shim: self.optimizer( @@ -196,7 +209,11 @@ async def _compile( ).run() args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] await _llvm.run( - "clang", args_o, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_o, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) return await self._parse(o) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 127d93b317fb09..5e1b05a3d86cb4 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -43,6 +43,9 @@ "--cflags", help="additional flags to pass to the compiler", default="" ) parser.add_argument("--llvm-version", help="LLVM version to use") + parser.add_argument( + "--llvm-tools-install-dir", help="Installation location of LLVM tools" + ) args = parser.parse_args() for target in args.target: target.debug = args.debug @@ -52,6 +55,8 @@ target.pyconfig_dir = args.pyconfig_dir if args.llvm_version: target.llvm_version = args.llvm_version + if args.llvm_tools_install_dir: + target.llvm_tools_install_dir = args.llvm_tools_install_dir target.build( comment=comment, force=args.force, diff --git a/configure b/configure index 562bb6860c79a9..49319bc2aa4459 100755 --- a/configure +++ b/configure @@ -11046,7 +11046,7 @@ then : else case e in #( e) as_fn_append CFLAGS_NODIST " $jit_flags" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\"" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\" --llvm-tools-install-dir=\"$LLVM_TOOLS_INSTALL_DIR\"" if test "x$Py_DEBUG" = xtrue then : as_fn_append REGEN_JIT_COMMAND " --debug" diff --git a/configure.ac b/configure.ac index 20e1afc2e9ee14..7b6f3c5e0ed5be 100644 --- a/configure.ac +++ b/configure.ac @@ -2850,7 +2850,7 @@ AS_VAR_IF([jit_flags], [], [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\""]) + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\" --llvm-tools-install-dir=\"$LLVM_TOOLS_INSTALL_DIR\""]) AS_VAR_IF([Py_DEBUG], [true], [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])], _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
