https://github.com/python/cpython/commit/c90863ac3dcbc5b0b8f9b5e3b0e5170aa23cffb7 commit: c90863ac3dcbc5b0b8f9b5e3b0e5170aa23cffb7 branch: main author: Savannah Ostrowski <[email protected]> committer: savannahostrowski <[email protected]> date: 2025-12-12T21:23:18Z summary:
GH-141362: Make `get_externals` handle fetching platform-specific release artifacts (#142405) Co-authored-by: Steve Dower <[email protected]> Co-authored-by: Chris Eibl <[email protected]> files: M PCbuild/get_external.py M Tools/jit/README.md M Tools/jit/jit_infra.md diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index edf14ce578bce6..27fbc311bbc1d6 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -3,6 +3,7 @@ import argparse import os import pathlib +import platform import sys import tarfile import time @@ -44,20 +45,38 @@ def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): def fetch_release(tag, tarball_dir, *, org='python', verbose=False): - url = f'https://github.com/{org}/cpython-bin-deps/releases/download/{tag}/{tag}.tar.xz' + arch = os.environ.get('PreferredToolArchitecture') + if not arch: + machine = platform.machine() + arch = 'ARM64' if machine == 'ARM64' else 'AMD64' + elif arch.lower() in ('x86', 'x64'): + arch = 'AMD64' reporthook = None if verbose: reporthook = print tarball_dir.mkdir(parents=True, exist_ok=True) - output_path = tarball_dir / f'{tag}.tar.xz' - retrieve_with_retries(url, output_path, reporthook) + + arch_filename = f'{tag}-{arch}.tar.xz' + arch_url = f'https://github.com/{org}/cpython-bin-deps/releases/download/{tag}/{arch_filename}' + try: + output_path = tarball_dir / arch_filename + retrieve_with_retries(arch_url, output_path, reporthook) + return output_path + except OSError: + if verbose: + print(f'{arch_filename} not found, trying generic binary...') + + generic_filename = f'{tag}.tar.xz' + generic_url = f'https://github.com/{org}/cpython-bin-deps/releases/download/{tag}/{generic_filename}' + output_path = tarball_dir / generic_filename + retrieve_with_retries(generic_url, output_path, reporthook) return output_path def extract_tarball(externals_dir, tarball_path, tag): output_path = externals_dir / tag with tarfile.open(tarball_path) as tf: - tf.extractall(os.fspath(externals_dir)) + tf.extractall(os.fspath(externals_dir), filter='data') return output_path diff --git a/Tools/jit/README.md b/Tools/jit/README.md index c70c0c47d94ad2..8eadb3349ba6da 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -43,6 +43,15 @@ Homebrew won't add any of the tools to your `$PATH`. That's okay; the build scri LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. +By default, the architecture of the LLVM tools is auto-detected based on the host machine. To override this, set the `PreferredToolArchitecture` environment variable before building: + +```sh +set PreferredToolArchitecture=AMD64 +PCbuild\build.bat --experimental-jit +``` + +Valid values are`x64`, `x86` and `ARM64`. + Otherwise, you can install LLVM 21 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=21), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): diff --git a/Tools/jit/jit_infra.md b/Tools/jit/jit_infra.md index 1a954755611d19..0b851c67d65c83 100644 --- a/Tools/jit/jit_infra.md +++ b/Tools/jit/jit_infra.md @@ -8,21 +8,28 @@ When we update LLVM, we need to also update the LLVM release artifact for Window To update the LLVM release artifact for Windows builds, follow these steps: 1. Go to the [LLVM releases page](https://github.com/llvm/llvm-project/releases). -1. Download x86_64 Windows artifact for the desired LLVM version (e.g. `clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz`). -1. Extract and repackage the tarball with the correct directory structure. For example: +1. Download Windows artifacts for the desired LLVM version (e.g. `clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz` and `clang+llvm-21.1.4-aarch64-pc-windows-msvc.tar.xz`). +1. Extract and repackage each tarball with the correct directory structure. For example: ```bash + # For x86_64 (AMD64) tar -xf clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz mv clang+llvm-21.1.4-x86_64-pc-windows-msvc llvm-21.1.4.0 - tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0.tar.xz + tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0-x64.tar.xz + rm -rf llvm-21.1.4.0 + + # For ARM64 + tar -xf clang+llvm-21.1.4-aarch64-pc-windows-msvc.tar.xz + mv clang+llvm-21.1.4-aarch64-pc-windows-msvc llvm-21.1.4.0 + tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0-ARM64.tar.xz ``` - The tarball must contain a top-level directory named `llvm-{version}.0/`. + Each tarball must contain a top-level directory named `llvm-{version}.0/`. 1. Go to [cpython-bin-deps](https://github.com/python/cpython-bin-deps). -1. Create a new release with the updated LLVM artifact. +1. Create a new release with the LLVM artifacts. - Create a new tag to match the LLVM version (e.g. `llvm-21.1.4.0`). - - Specify the release title (e.g. `LLVM 21.1.4 for x86_64 Windows`). - - Upload the asset (you can leave all other fields the same). + - Specify the release title (e.g. `LLVM 21.1.4`). + - Upload both platform-specific assets to the same release. ### Other notes - You must make sure that the name of the artifact matches exactly what is expected in `Tools/jit/_llvm.py` and `PCbuild/get_externals.py`. -- We don't need multiple release artifacts for each architecture because LLVM can cross-compile for different architectures on Windows; x86_64 is sufficient. +- The artifact filename must include the architecture suffix (e.g. `llvm-21.1.4.0-x64.tar.xz`, `llvm-21.1.4.0-ARM64.tar.xz`). - You must have permissions to create releases in the `cpython-bin-deps` repository. If you don't have permissions, you should contact one of the organization admins. \ No newline at end of file _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
