https://github.com/python/cpython/commit/ebb150e76ab4988fdcd5e8caa36b9014497573a5 commit: ebb150e76ab4988fdcd5e8caa36b9014497573a5 branch: main author: Hood Chatham <[email protected]> committer: freakboy3742 <[email protected]> date: 2026-03-11T08:43:27+08:00 summary:
gh-145219: Cache Emscripten build dependencies, add install-emscripten (#145664) Modifies the Emscripten build script to allow for caching of dependencies, and for automated installation of new EMSDK versions. Co-authored-by: Russell Keith-Magee <[email protected]> files: A Tools/wasm/emscripten/config.toml D Tools/wasm/emscripten/emscripten_version.txt M Tools/wasm/emscripten/__main__.py diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 14d32279a8c4fa..b1a779777ae9fc 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -4,6 +4,7 @@ import contextlib import functools import hashlib +import json import os import shutil import subprocess @@ -14,6 +15,8 @@ from textwrap import dedent from urllib.request import urlopen +import tomllib + try: from os import process_cpu_count as cpu_count except ImportError: @@ -22,25 +25,51 @@ EMSCRIPTEN_DIR = Path(__file__).parent CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent -EMSCRIPTEN_VERSION_FILE = EMSCRIPTEN_DIR / "emscripten_version.txt" +CONFIG_FILE = EMSCRIPTEN_DIR / "config.toml" DEFAULT_CROSS_BUILD_DIR = CHECKOUT / "cross-build" HOST_TRIPLE = "wasm32-emscripten" -def get_build_paths(cross_build_dir=None): [email protected] +def load_config_toml(): + with CONFIG_FILE.open("rb") as file: + return tomllib.load(file) + + [email protected] +def required_emscripten_version(): + return load_config_toml()["emscripten-version"] + + [email protected] +def emsdk_cache_root(emsdk_cache): + required_version = required_emscripten_version() + return Path(emsdk_cache).absolute() / required_version + + [email protected] +def emsdk_activate_path(emsdk_cache): + return emsdk_cache_root(emsdk_cache) / "emsdk/emsdk_env.sh" + + +def get_build_paths(cross_build_dir=None, emsdk_cache=None): """Compute all build paths from the given cross-build directory.""" if cross_build_dir is None: cross_build_dir = DEFAULT_CROSS_BUILD_DIR cross_build_dir = Path(cross_build_dir).absolute() host_triple_dir = cross_build_dir / HOST_TRIPLE + prefix_dir = host_triple_dir / "prefix" + if emsdk_cache: + prefix_dir = emsdk_cache_root(emsdk_cache) / "prefix" + return { "cross_build_dir": cross_build_dir, "native_build_dir": cross_build_dir / "build", "host_triple_dir": host_triple_dir, "host_build_dir": host_triple_dir / "build", "host_dir": host_triple_dir / "build" / "python", - "prefix_dir": host_triple_dir / "prefix", + "prefix_dir": prefix_dir, } @@ -48,22 +77,10 @@ def get_build_paths(cross_build_dir=None): LOCAL_SETUP_MARKER = b"# Generated by Tools/wasm/emscripten.py\n" [email protected] -def get_required_emscripten_version(): - """Read the required emscripten version from emscripten_version.txt.""" - return EMSCRIPTEN_VERSION_FILE.read_text().strip() - - [email protected] -def get_emsdk_activate_path(emsdk_cache): - required_version = get_required_emscripten_version() - return Path(emsdk_cache) / required_version / "emsdk_env.sh" - - def validate_emsdk_version(emsdk_cache): """Validate that the emsdk cache contains the required emscripten version.""" - required_version = get_required_emscripten_version() - emsdk_env = get_emsdk_activate_path(emsdk_cache) + required_version = required_emscripten_version() + emsdk_env = emsdk_activate_path(emsdk_cache) if not emsdk_env.is_file(): print( f"Required emscripten version {required_version} not found in {emsdk_cache}", @@ -90,7 +107,7 @@ def get_emsdk_environ(emsdk_cache): [ "bash", "-c", - f"EMSDK_QUIET=1 source {get_emsdk_activate_path(emsdk_cache)} && env", + f"EMSDK_QUIET=1 source {emsdk_activate_path(emsdk_cache)} && env", ], text=True, ) @@ -207,6 +224,35 @@ def build_python_path(context): return binary +def install_emscripten(context): + emsdk_cache = context.emsdk_cache + if emsdk_cache is None: + print("install-emscripten requires --emsdk-cache", file=sys.stderr) + sys.exit(1) + version = required_emscripten_version() + emsdk_target = emsdk_cache_root(emsdk_cache) / "emsdk" + if emsdk_target.exists(): + if not context.quiet: + print(f"Emscripten version {version} already installed") + return + if not context.quiet: + print(f"Installing emscripten version {version}") + emsdk_target.mkdir(parents=True) + call( + [ + "git", + "clone", + "https://github.com/emscripten-core/emsdk.git";, + emsdk_target, + ], + quiet=context.quiet, + ) + call([emsdk_target / "emsdk", "install", version], quiet=context.quiet) + call([emsdk_target / "emsdk", "activate", version], quiet=context.quiet) + if not context.quiet: + print(f"Installed emscripten version {version}") + + @subdir("native_build_dir", clean_ok=True) def configure_build_python(context, working_dir): """Configure the build/host Python.""" @@ -258,35 +304,87 @@ def download_and_unpack(working_dir: Path, url: str, expected_shasum: str): shutil.unpack_archive(tmp_file.name, working_dir) +def should_build_library(prefix, name, config, quiet): + cached_config = prefix / (name + ".json") + if not cached_config.exists(): + if not quiet: + print( + f"No cached build of {name} version {config['version']} found, building" + ) + return True + + try: + with cached_config.open("rb") as f: + cached_config = json.load(f) + except json.JSONDecodeError: + if not quiet: + print(f"Cached data for {name} invalid, rebuilding") + return True + if config == cached_config: + if not quiet: + print( + f"Found cached build of {name} version {config['version']}, not rebuilding" + ) + return False + + if not quiet: + print( + f"Found cached build of {name} version {config['version']} but it's out of date, rebuilding" + ) + return True + + +def write_library_config(prefix, name, config, quiet): + cached_config = prefix / (name + ".json") + with cached_config.open("w") as f: + json.dump(config, f) + if not quiet: + print(f"Succeded building {name}, wrote config to {cached_config}") + + @subdir("host_build_dir", clean_ok=True) def make_emscripten_libffi(context, working_dir): - ver = "3.4.6" - libffi_dir = working_dir / f"libffi-{ver}" + prefix = context.build_paths["prefix_dir"] + libffi_config = load_config_toml()["libffi"] + if not should_build_library( + prefix, "libffi", libffi_config, context.quiet + ): + return + url = libffi_config["url"] + version = libffi_config["version"] + shasum = libffi_config["shasum"] + libffi_dir = working_dir / f"libffi-{version}" shutil.rmtree(libffi_dir, ignore_errors=True) download_and_unpack( working_dir, - f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz";, - "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e", + url.format(version=version), + shasum, ) call( [EMSCRIPTEN_DIR / "make_libffi.sh"], - env=updated_env( - {"PREFIX": context.build_paths["prefix_dir"]}, context.emsdk_cache - ), + env=updated_env({"PREFIX": prefix}, context.emsdk_cache), cwd=libffi_dir, quiet=context.quiet, ) + write_library_config(prefix, "libffi", libffi_config, context.quiet) @subdir("host_build_dir", clean_ok=True) def make_mpdec(context, working_dir): - ver = "4.0.1" - mpdec_dir = working_dir / f"mpdecimal-{ver}" + prefix = context.build_paths["prefix_dir"] + mpdec_config = load_config_toml()["mpdec"] + if not should_build_library(prefix, "mpdec", mpdec_config, context.quiet): + return + + url = mpdec_config["url"] + version = mpdec_config["version"] + shasum = mpdec_config["shasum"] + mpdec_dir = working_dir / f"mpdecimal-{version}" shutil.rmtree(mpdec_dir, ignore_errors=True) download_and_unpack( working_dir, - f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz";, - "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8", + url.format(version=version), + shasum, ) call( [ @@ -294,7 +392,7 @@ def make_mpdec(context, working_dir): mpdec_dir / "configure", "CFLAGS=-fPIC", "--prefix", - context.build_paths["prefix_dir"], + prefix, "--disable-shared", ], cwd=mpdec_dir, @@ -306,6 +404,7 @@ def make_mpdec(context, working_dir): cwd=mpdec_dir, quiet=context.quiet, ) + write_library_config(prefix, "mpdec", mpdec_config, context.quiet) @subdir("host_dir", clean_ok=True) @@ -436,16 +535,24 @@ def make_emscripten_python(context, working_dir): subprocess.check_call([exec_script, "--version"]) -def build_all(context): - """Build everything.""" - steps = [ - configure_build_python, - make_build_python, - make_emscripten_libffi, - make_mpdec, - configure_emscripten_python, - make_emscripten_python, - ] +def build_target(context): + """Build one or more targets.""" + steps = [] + if context.target in {"all"}: + steps.append(install_emscripten) + if context.target in {"build", "all"}: + steps.extend([ + configure_build_python, + make_build_python, + ]) + if context.target in {"host", "all"}: + steps.extend([ + make_emscripten_libffi, + make_mpdec, + configure_emscripten_python, + make_emscripten_python, + ]) + for step in steps: step(context) @@ -475,7 +582,22 @@ def main(): parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") + install_emscripten_cmd = subcommands.add_parser( + "install-emscripten", + help="Install the appropriate version of Emscripten", + ) build = subcommands.add_parser("build", help="Build everything") + build.add_argument( + "target", + nargs="?", + default="all", + choices=["all", "host", "build"], + help=( + "What should be built. 'build' for just the build platform, or " + "'host' for the host platform, or 'all' for both. Defaults to 'all'." + ), + ) + configure_build = subcommands.add_parser( "configure-build-python", help="Run `configure` for the build Python" ) @@ -512,6 +634,7 @@ def main(): ) for subcommand in ( + install_emscripten_cmd, build, configure_build, make_libffi_cmd, @@ -568,22 +691,25 @@ def main(): context = parser.parse_args() - context.build_paths = get_build_paths(context.cross_build_dir) - - if context.emsdk_cache: + if context.emsdk_cache and context.subcommand != "install-emscripten": validate_emsdk_version(context.emsdk_cache) context.emsdk_cache = Path(context.emsdk_cache).absolute() else: print("Build will use EMSDK from current environment.") + context.build_paths = get_build_paths( + context.cross_build_dir, context.emsdk_cache + ) + dispatch = { + "install-emscripten": install_emscripten, "make-libffi": make_emscripten_libffi, "make-mpdec": make_mpdec, "configure-build-python": configure_build_python, "make-build-python": make_build_python, "configure-host": configure_emscripten_python, "make-host": make_emscripten_python, - "build": build_all, + "build": build_target, "clean": clean_contents, } diff --git a/Tools/wasm/emscripten/config.toml b/Tools/wasm/emscripten/config.toml new file mode 100644 index 00000000000000..98edaebe992685 --- /dev/null +++ b/Tools/wasm/emscripten/config.toml @@ -0,0 +1,14 @@ +# Any data that can vary between Python versions is to be kept in this file. +# This allows for blanket copying of the Emscripten build code between supported +# Python versions. +emscripten-version = "4.0.12" + +[libffi] +url = "https://github.com/libffi/libffi/releases/download/v{version}/libffi-{version}.tar.gz"; +version = "3.4.6" +shasum = "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e" + +[mpdec] +url = "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{version}.tar.gz"; +version = "4.0.1" +shasum = "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8" diff --git a/Tools/wasm/emscripten/emscripten_version.txt b/Tools/wasm/emscripten/emscripten_version.txt deleted file mode 100644 index 4c05e4ef57dbf8..00000000000000 --- a/Tools/wasm/emscripten/emscripten_version.txt +++ /dev/null @@ -1 +0,0 @@ -4.0.12 _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
