https://github.com/python/cpython/commit/bfcbb28223b733b9cb88f152a059a9e1416f3467
commit: bfcbb28223b733b9cb88f152a059a9e1416f3467
branch: main
author: Brandt Bucher <brandtbuc...@microsoft.com>
committer: brandtbucher <brandtbuc...@gmail.com>
date: 2025-05-02T11:17:15-07:00
summary:
GH-113464: Get LLVM from cpython-bin-deps on Windows (GH-133278)
files:
A Misc/NEWS.d/next/Build/2025-05-01-17-27-06.gh-issue-113464.vjE5X4.rst
M .github/workflows/jit.yml
M PCbuild/build.bat
M PCbuild/get_externals.bat
M PCbuild/regen.targets
M Tools/jit/README.md
M Tools/jit/_llvm.py
M Tools/jit/_targets.py
M Tools/jit/build.py
diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml
index 6ea238d294ca89..116e0c1e945e38 100644
--- a/.github/workflows/jit.yml
+++ b/.github/workflows/jit.yml
@@ -95,10 +95,10 @@ jobs:
with:
python-version: '3.11'
+ # PCbuild downloads LLVM automatically:
- name: Windows
if: runner.os == 'Windows'
run: |
- choco install llvm --allow-downgrade --no-progress --version ${{
matrix.llvm }}.1.0
./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' ||
'' }} -p ${{ matrix.architecture }}
./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{
matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
diff --git
a/Misc/NEWS.d/next/Build/2025-05-01-17-27-06.gh-issue-113464.vjE5X4.rst
b/Misc/NEWS.d/next/Build/2025-05-01-17-27-06.gh-issue-113464.vjE5X4.rst
new file mode 100644
index 00000000000000..d150b19d5f1687
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2025-05-01-17-27-06.gh-issue-113464.vjE5X4.rst
@@ -0,0 +1,3 @@
+Use the cpython-bin-deps "externals" repository for Windows LLVM dependency
+management. Installing LLVM manually is no longer necessary for Windows JIT
+builds.
diff --git a/PCbuild/build.bat b/PCbuild/build.bat
index 88d6b99d651cf3..2f358991e484ce 100644
--- a/PCbuild/build.bat
+++ b/PCbuild/build.bat
@@ -111,6 +111,7 @@ if "%IncludeExternals%"=="" set IncludeExternals=true
if "%IncludeCTypes%"=="" set IncludeCTypes=true
if "%IncludeSSL%"=="" set IncludeSSL=true
if "%IncludeTkinter%"=="" set IncludeTkinter=true
+if "%UseJIT%" NEQ "true" set IncludeLLVM=false
if "%IncludeExternals%"=="true" call "%dir%get_externals.bat"
diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
index b28ce9caeb3f13..49ace616793d8e 100644
--- a/PCbuild/get_externals.bat
+++ b/PCbuild/get_externals.bat
@@ -15,6 +15,7 @@ set IncludeSSLSrc=false
if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts
if "%~1"=="--no-libffi" (set IncludeLibffi=false) & shift & goto CheckOpts
+if "%~1"=="--no-llvm" (set IncludeLLVM=false) & shift & goto CheckOpts
if "%~1"=="--tkinter-src" (set IncludeTkinterSrc=true) & shift & goto CheckOpts
if "%~1"=="--openssl-src" (set IncludeSSLSrc=true) & shift & goto CheckOpts
if "%~1"=="--libffi-src" (set IncludeLibffiSrc=true) & shift & goto CheckOpts
@@ -80,6 +81,7 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries%
libffi-3.4.4
if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.16.2
if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0
if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06
+if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-19.1.7.0
for %%b in (%binaries%) do (
if exist "%EXTERNALS_DIR%\%%b" (
@@ -98,7 +100,7 @@ goto end
:usage
echo.Valid options: -c, --clean, --clean-only, --organization, --python,
-echo.--no-tkinter, --no-openssl
+echo.--no-tkinter, --no-openssl, --no-llvm
echo.
echo.Pull all sources and binaries necessary for compiling optional extension
echo.modules that rely on external libraries.
diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets
index 416241d9d0df10..e7822a126c6304 100644
--- a/PCbuild/regen.targets
+++ b/PCbuild/regen.targets
@@ -30,7 +30,11 @@
<_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" />
<!-- Taken from _Target._compute_digest in Tools\jit\_targets.py: -->
<_JITSources
Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/>
+ <!-- Need to explicitly enumerate these, since globbing doesn't work for
missing outputs: -->
<_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/>
+ <_JITOutputs
Include="$(GeneratedPyConfigDir)jit_stencils-aarch64-pc-windows-msvc.h"
Condition="$(Platform) == 'ARM64'"/>
+ <_JITOutputs
Include="$(GeneratedPyConfigDir)jit_stencils-i686-pc-windows-msvc.h"
Condition="$(Platform) == 'Win32'"/>
+ <_JITOutputs
Include="$(GeneratedPyConfigDir)jit_stencils-x86_64-pc-windows-msvc.h"
Condition="$(Platform) == 'x64'"/>
<_CasesSources
Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/>
<_CasesOutputs
Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/>
<_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" />
@@ -124,6 +128,9 @@
<Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py"
$(JITArgs)'
WorkingDirectory="$(GeneratedPyConfigDir)"/>
</Target>
+ <Target Name="_CleanJIT" AfterTargets="Clean">
+ <Delete Files="@(_JITOutputs)"/>
+ </Target>
<Target Name="_RegenNoPGUpdate"
Condition="$(Configuration) != 'PGUpdate'"
diff --git a/Tools/jit/README.md b/Tools/jit/README.md
index 4107265754f6ec..ff4b3964e65bad 100644
--- a/Tools/jit/README.md
+++ b/Tools/jit/README.md
@@ -41,7 +41,9 @@ Homebrew won't add any of the tools to your `$PATH`. That's
okay; the build scri
### Windows
-Install LLVM 19 [by searching for it on LLVM's GitHub releases
page](https://github.com/llvm/llvm-project/releases?q=19), clicking on
"Assets", downloading the appropriate Windows installer for your platform
(likely the file ending with `-win64.exe`), and running it. **When installing,
be sure to select the option labeled "Add LLVM to the system PATH".**
+LLVM is downloaded automatically (along with other external binary
dependencies) by `PCbuild\build.bat`.
+
+Otherwise, you can install LLVM 19 [by searching for it on LLVM's GitHub
releases page](https://github.com/llvm/llvm-project/releases?q=19), clicking on
"Assets", downloading the appropriate Windows installer for your platform
(likely the file ending with `-win64.exe`), and running it. **When installing,
be sure to select the option labeled "Add LLVM to the system PATH".**
Alternatively, you can use [chocolatey](https://chocolatey.org):
diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py
index 925b56ac669aa0..f09a8404871b24 100644
--- a/Tools/jit/_llvm.py
+++ b/Tools/jit/_llvm.py
@@ -8,8 +8,11 @@
import subprocess
import typing
+import _targets
+
_LLVM_VERSION = 19
_LLVM_VERSION_PATTERN =
re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+")
+_EXTERNALS_LLVM_TAG = "llvm-19.1.7.0"
_P = typing.ParamSpec("_P")
_R = typing.TypeVar("_R")
@@ -72,6 +75,11 @@ async def _find_tool(tool: str, *, echo: bool = False) ->
str | None:
return path
# Versioned executables:
path = f"{tool}-{_LLVM_VERSION}"
+ if await _check_tool_version(path, echo=echo):
+ return path
+ # PCbuild externals:
+ externals = os.environ.get("EXTERNALS_DIR", _targets.EXTERNALS)
+ path = os.path.join(externals, _EXTERNALS_LLVM_TAG, "bin", tool)
if await _check_tool_version(path, echo=echo):
return path
# Homebrew-installed executables:
diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py
index e0ac735a20f691..6ceb4404e74ce7 100644
--- a/Tools/jit/_targets.py
+++ b/Tools/jit/_targets.py
@@ -23,8 +23,10 @@
TOOLS_JIT = TOOLS_JIT_BUILD.parent
TOOLS = TOOLS_JIT.parent
CPYTHON = TOOLS.parent
+EXTERNALS = CPYTHON / "externals"
PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h"
TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c"
+
ASYNCIO_RUNNER = asyncio.Runner()
_S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection,
_schema.MachOSection)
diff --git a/Tools/jit/build.py b/Tools/jit/build.py
index 4d1e484b6838eb..49b08f477dbed7 100644
--- a/Tools/jit/build.py
+++ b/Tools/jit/build.py
@@ -8,6 +8,7 @@
import _targets
if __name__ == "__main__":
+ out = pathlib.Path.cwd().resolve()
comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}"
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
@@ -31,17 +32,22 @@
target.force = args.force
target.verbose = args.verbose
target.build(
- pathlib.Path.cwd(),
+ out,
comment=comment,
stencils_h=f"jit_stencils-{target.triple}.h",
force=args.force,
)
-
- with open("jit_stencils.h", "w") as fp:
- for idx, target in enumerate(args.target):
- fp.write(f"#{'if' if idx == 0 else 'elif'} {target.condition}\n")
- fp.write(f'#include "jit_stencils-{target.triple}.h"\n')
-
- fp.write("#else\n")
- fp.write('#error "unexpected target"\n')
- fp.write("#endif\n")
+ jit_stencils_h = out / "jit_stencils.h"
+ lines = [f"// {comment}\n"]
+ guard = "#if"
+ for target in args.target:
+ lines.append(f"{guard} {target.condition}\n")
+ lines.append(f'#include "jit_stencils-{target.triple}.h"\n')
+ guard = "#elif"
+ lines.append("#else\n")
+ lines.append('#error "unexpected target"\n')
+ lines.append("#endif\n")
+ body = "".join(lines)
+ # Don't touch the file if it hasn't changed (so we don't trigger a
rebuild):
+ if not jit_stencils_h.is_file() or jit_stencils_h.read_text() != body:
+ jit_stencils_h.write_text(body)
_______________________________________________
Python-checkins mailing list -- python-checkins@python.org
To unsubscribe send an email to python-checkins-le...@python.org
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: arch...@mail-archive.com
