https://github.com/python/cpython/commit/b63dc8abdf7b985bb23d212862baf29736888394 commit: b63dc8abdf7b985bb23d212862baf29736888394 branch: main author: Brett Cannon <[email protected]> committer: brettcannon <[email protected]> date: 2026-03-04T13:27:23-08:00 summary:
Refactor `Platforms/WASI/__main__.py` for lazy importing and future new subcommands (#145404) Co-authored-by: Hugo van Kemenade <[email protected]> files: A Platforms/WASI/_build.py M .pre-commit-config.yaml M Platforms/WASI/.ruff.toml M Platforms/WASI/__main__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1dcb50e31d9a68..4893ec28f23e5a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,10 @@ repos: name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] files: ^Lib/test/ + - id: ruff-check + name: Run Ruff (lint) on Platforms/WASI/ + args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml] + files: ^Platforms/WASI/ - id: ruff-check name: Run Ruff (lint) on Tools/build/ args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] @@ -42,6 +46,10 @@ repos: name: Run Ruff (format) on Doc/ args: [--exit-non-zero-on-fix] files: ^Doc/ + - id: ruff-format + name: Run Ruff (format) on Platforms/WASI/ + args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml] + files: ^Platforms/WASI/ - id: ruff-format name: Run Ruff (format) on Tools/build/check_warnings.py args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] diff --git a/Platforms/WASI/.ruff.toml b/Platforms/WASI/.ruff.toml index 3d8e59fa3f22c4..492713c1520000 100644 --- a/Platforms/WASI/.ruff.toml +++ b/Platforms/WASI/.ruff.toml @@ -1,5 +1,7 @@ extend = "../../.ruff.toml" # Inherit the project-wide settings +target-version = "py314" + [format] preview = true docstring-code-format = true diff --git a/Platforms/WASI/__main__.py b/Platforms/WASI/__main__.py index 471ac3297b2702..b8513a004f18e5 100644 --- a/Platforms/WASI/__main__.py +++ b/Platforms/WASI/__main__.py @@ -1,417 +1,23 @@ #!/usr/bin/env python3 +__lazy_modules__ = ["_build"] + import argparse -import contextlib -import functools import os - -import tomllib - -try: - from os import process_cpu_count as cpu_count -except ImportError: - from os import cpu_count import pathlib -import shutil -import subprocess -import sys -import sysconfig -import tempfile - -CHECKOUT = HERE = pathlib.Path(__file__).parent - -while CHECKOUT != CHECKOUT.parent: - if (CHECKOUT / "configure").is_file(): - break - CHECKOUT = CHECKOUT.parent -else: - raise FileNotFoundError( - "Unable to find the root of the CPython checkout by looking for 'configure'" - ) - -CROSS_BUILD_DIR = CHECKOUT / "cross-build" -# Build platform can also be found via `config.guess`. -BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") - -LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = ( - b"# Generated by Platforms/WASI .\n" - b"# Required to statically build extension modules." -) - -WASI_SDK_VERSION = 29 - -WASMTIME_VAR_NAME = "WASMTIME" -WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" - - -def separator(): - """Print a separator line across the terminal width.""" - try: - tput_output = subprocess.check_output( - ["tput", "cols"], encoding="utf-8" - ) - except subprocess.CalledProcessError: - terminal_width = 80 - else: - terminal_width = int(tput_output.strip()) - print("โฏ" * terminal_width) - - -def log(emoji, message, *, spacing=None): - """Print a notification with an emoji. - - If 'spacing' is None, calculate the spacing based on the number of code points - in the emoji as terminals "eat" a space when the emoji has multiple code points. - """ - if spacing is None: - spacing = " " if len(emoji) == 1 else " " - print("".join([emoji, spacing, message])) - - -def updated_env(updates={}): - """Create a new dict representing the environment to use. - - The changes made to the execution environment are printed out. - """ - env_defaults = {} - # https://reproducible-builds.org/docs/source-date-epoch/ - git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] - try: - epoch = subprocess.check_output( - git_epoch_cmd, encoding="utf-8" - ).strip() - env_defaults["SOURCE_DATE_EPOCH"] = epoch - except subprocess.CalledProcessError: - pass # Might be building from a tarball. - # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. - environment = env_defaults | os.environ | updates - - env_diff = {} - for key, value in environment.items(): - if os.environ.get(key) != value: - env_diff[key] = value - - env_vars = ( - f"\n {key}={item}" for key, item in sorted(env_diff.items()) - ) - log("๐", f"Environment changes:{''.join(env_vars)}") - - return environment - - -def subdir(working_dir, *, clean_ok=False): - """Decorator to change to a working directory.""" - - def decorator(func): - @functools.wraps(func) - def wrapper(context): - nonlocal working_dir - - if callable(working_dir): - working_dir = working_dir(context) - separator() - log("๐", os.fsdecode(working_dir)) - if ( - clean_ok - and getattr(context, "clean", False) - and working_dir.exists() - ): - log("๐ฎ", "Deleting directory (--clean)...") - shutil.rmtree(working_dir) - - working_dir.mkdir(parents=True, exist_ok=True) - - with contextlib.chdir(working_dir): - return func(context, working_dir) - - return wrapper - - return decorator - - -def call(command, *, context=None, quiet=False, logdir=None, **kwargs): - """Execute a command. - - If 'quiet' is true, then redirect stdout and stderr to a temporary file. - """ - if context is not None: - quiet = context.quiet - logdir = context.logdir - elif quiet and logdir is None: - raise ValueError("When quiet is True, logdir must be specified") - - log("โฏ", " ".join(map(str, command)), spacing=" ") - if not quiet: - stdout = None - stderr = None - else: - stdout = tempfile.NamedTemporaryFile( - "w", - encoding="utf-8", - delete=False, - dir=logdir, - prefix="cpython-wasi-", - suffix=".log", - ) - stderr = subprocess.STDOUT - log("๐", f"Logging output to {stdout.name} (--quiet)...") - - subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) - - -def build_python_path(): - """The path to the build Python binary.""" - binary = BUILD_DIR / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError( - f"Unable to find `python(.exe)` in {BUILD_DIR}" - ) - - return binary - - -def build_python_is_pydebug(): - """Find out if the build Python is a pydebug build.""" - test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" - result = subprocess.run( - [build_python_path(), "-c", test], - capture_output=True, - ) - return bool(result.returncode) +import _build -@subdir(BUILD_DIR, clean_ok=True) -def configure_build_python(context, working_dir): - """Configure the build/host Python.""" - if LOCAL_SETUP.exists(): - if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: - log("๐", f"{LOCAL_SETUP} exists ...") - else: - log("โ ๏ธ", f"{LOCAL_SETUP} exists, but has unexpected contents") - else: - log("๐", f"Creating {LOCAL_SETUP} ...") - LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) - - configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] - if context.args: - configure.extend(context.args) - - call(configure, context=context) - - -@subdir(BUILD_DIR) -def make_build_python(context, working_dir): - """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], context=context) - - binary = build_python_path() - cmd = [ - binary, - "-c", - "import sys; " - "print(f'{sys.version_info.major}.{sys.version_info.minor}')", - ] - version = subprocess.check_output(cmd, encoding="utf-8").strip() - - log("๐", f"{binary} {version}") - - -def find_wasi_sdk(config): - """Find the path to the WASI SDK.""" - wasi_sdk_path = None - wasi_sdk_version = config["targets"]["wasi-sdk"] - - if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"): - wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) - else: - opt_path = pathlib.Path("/opt") - # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team - # has said they don't plan to ever do a point release and all of their Git tags - # lack the ``.0`` suffix. - # Starting with WASI SDK 23, the tarballs went from containing a directory named - # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. - # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. - potential_sdks = [ - path - for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*") - if path.is_dir() - ] - if len(potential_sdks) == 1: - wasi_sdk_path = potential_sdks[0] - elif (default_path := opt_path / "wasi-sdk").is_dir(): - wasi_sdk_path = default_path - - # Starting with WASI SDK 25, a VERSION file is included in the root - # of the SDK directory that we can read to warn folks when they are using - # an unsupported version. - if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file(): - version_details = version_file.read_text(encoding="utf-8") - found_version = version_details.splitlines()[0] - # Make sure there's a trailing dot to avoid false positives if somehow the - # supported version is a prefix of the found version (e.g. `25` and `2567`). - if not found_version.startswith(f"{wasi_sdk_version}."): - major_version = found_version.partition(".")[0] - log( - "โ ๏ธ", - f" Found WASI SDK {major_version}, " - f"but WASI SDK {wasi_sdk_version} is the supported version", - ) - - return wasi_sdk_path - - -def wasi_sdk_env(context): - """Calculate environment variables for building with wasi-sdk.""" - wasi_sdk_path = context.wasi_sdk_path - sysroot = wasi_sdk_path / "share" / "wasi-sysroot" - env = { - "CC": "clang", - "CPP": "clang-cpp", - "CXX": "clang++", - "AR": "llvm-ar", - "RANLIB": "ranlib", - } - - for env_var, binary_name in list(env.items()): - env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) - - if not wasi_sdk_path.name.startswith("wasi-sdk"): - for compiler in ["CC", "CPP", "CXX"]: - env[compiler] += f" --sysroot={sysroot}" - - env["PKG_CONFIG_PATH"] = "" - env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( - map( - os.fsdecode, - [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], - ) - ) - env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) - - env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) - env["WASI_SYSROOT"] = os.fsdecode(sysroot) - - env["PATH"] = os.pathsep.join([ - os.fsdecode(wasi_sdk_path / "bin"), - os.environ["PATH"], - ]) - - return env - - -@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) -def configure_wasi_python(context, working_dir): - """Configure the WASI/host build.""" - if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError( - "WASI-SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk" - ) - - config_site = os.fsdecode(HERE / "config.site-wasm32-wasi") - - wasi_build_dir = working_dir.relative_to(CHECKOUT) - - args = { - "ARGV0": f"/{wasi_build_dir}/python.wasm", - "PYTHON_WASM": working_dir / "python.wasm", - } - # Check dynamically for wasmtime in case it was specified manually via - # `--host-runner`. - if WASMTIME_HOST_RUNNER_VAR in context.host_runner: - if wasmtime := shutil.which("wasmtime"): - args[WASMTIME_VAR_NAME] = wasmtime - else: - raise FileNotFoundError( - "wasmtime not found; download from " - "https://github.com/bytecodealliance/wasmtime"; - ) - host_runner = context.host_runner.format_map(args) - env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} - build_python = os.fsdecode(build_python_path()) - # The path to `configure` MUST be relative, else `python.wasm` is unable - # to find the stdlib due to Python not recognizing that it's being - # executed from within a checkout. - configure = [ - os.path.relpath(CHECKOUT / "configure", working_dir), - f"--host={context.host_triple}", - f"--build={BUILD_DIR.name}", - f"--with-build-python={build_python}", - ] - if build_python_is_pydebug(): - configure.append("--with-pydebug") - if context.args: - configure.extend(context.args) - call( - configure, - env=updated_env(env_additions | wasi_sdk_env(context)), - context=context, - ) - - python_wasm = working_dir / "python.wasm" - exec_script = working_dir / "python.sh" - with exec_script.open("w", encoding="utf-8") as file: - file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') - exec_script.chmod(0o755) - log("๐", f"Created {exec_script} (--host-runner)... ") - sys.stdout.flush() - - -@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) -def make_wasi_python(context, working_dir): - """Run `make` for the WASI/host build.""" - call( - ["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - context=context, - ) - - exec_script = working_dir / "python.sh" - call([exec_script, "--version"], quiet=False) - log( - "๐", - f"Use `{exec_script.relative_to(context.init_dir)}` " - "to run CPython w/ the WASI host specified by --host-runner", - ) - - -def clean_contents(context): - """Delete all files created by this script.""" - if CROSS_BUILD_DIR.exists(): - log("๐งน", f"Deleting {CROSS_BUILD_DIR} ...") - shutil.rmtree(CROSS_BUILD_DIR) - - if LOCAL_SETUP.exists(): - if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: - log("๐งน", f"Deleting generated {LOCAL_SETUP} ...") - - -def build_steps(*steps): - """Construct a command from other steps.""" - - def builder(context): - for step in steps: - step(context) - - return builder +HERE = pathlib.Path(__file__).parent def main(): - with (HERE / "config.toml").open("rb") as file: - config = tomllib.load(file) - default_wasi_sdk = find_wasi_sdk(config) - default_host_triple = config["targets"]["host-triple"] default_host_runner = ( - f"{WASMTIME_HOST_RUNNER_VAR} run " - # Set argv0 so that getpath.py can auto-discover the sysconfig data directory + "{WASMTIME} run " + # Set argv0 so that getpath.py can auto-discover the sysconfig data directory. "--argv0 {ARGV0} " # Map the checkout to / to load the stdlib from /Lib. - f"--dir {os.fsdecode(CHECKOUT)}::/ " + "--dir {CHECKOUT}::/ " # Flags involving --optimize, --codegen, --debug, --wasm, and --wasi can be kept # in a config file. # We are using such a file to act as defaults in case a user wants to override @@ -419,9 +25,8 @@ def main(): # post-build so that they immediately apply to the Makefile instead of having to # regenerate it, and allow for easy copying of the settings for anyone else who # may want to use them. - f"--config {os.fsdecode(HERE / 'wasmtime.toml')}" + "--config {WASMTIME_CONFIG_PATH}" ) - default_logdir = pathlib.Path(tempfile.gettempdir()) parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") @@ -470,8 +75,8 @@ def main(): subcommand.add_argument( "--logdir", type=pathlib.Path, - default=default_logdir, - help=f"Directory to store log files; defaults to {default_logdir}", + default=None, + help="Directory to store log files", ) for subcommand in ( configure_build, @@ -501,8 +106,9 @@ def main(): "--wasi-sdk", type=pathlib.Path, dest="wasi_sdk_path", - default=default_wasi_sdk, - help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}", + default=None, + help="Path to the WASI SDK; defaults to WASI_SDK_PATH environment variable " + "or the appropriate version found in /opt", ) subcommand.add_argument( "--host-runner", @@ -516,28 +122,37 @@ def main(): subcommand.add_argument( "--host-triple", action="store", - default=default_host_triple, + default=None, help="The target triple for the WASI host build; " - f"defaults to {default_host_triple}", + f"defaults to the value found in {os.fsdecode(HERE / 'config.toml')}", ) context = parser.parse_args() - context.init_dir = pathlib.Path().absolute() - - build_build_python = build_steps(configure_build_python, make_build_python) - build_wasi_python = build_steps(configure_wasi_python, make_wasi_python) - dispatch = { - "configure-build-python": configure_build_python, - "make-build-python": make_build_python, - "build-python": build_build_python, - "configure-host": configure_wasi_python, - "make-host": make_wasi_python, - "build-host": build_wasi_python, - "build": build_steps(build_build_python, build_wasi_python), - "clean": clean_contents, - } - dispatch[context.subcommand](context) + match context.subcommand: + case "configure-build-python": + _build.configure_build_python(context) + case "make-build-python": + _build.make_build_python(context) + case "build-python": + _build.configure_build_python(context) + _build.make_build_python(context) + case "configure-host": + _build.configure_wasi_python(context) + case "make-host": + _build.make_wasi_python(context) + case "build-host": + _build.configure_wasi_python(context) + _build.make_wasi_python(context) + case "build": + _build.configure_build_python(context) + _build.make_build_python(context) + _build.configure_wasi_python(context) + _build.make_wasi_python(context) + case "clean": + _build.clean_contents(context) + case _: + raise ValueError(f"Unknown subcommand {context.subcommand!r}") if __name__ == "__main__": diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py new file mode 100644 index 00000000000000..76d2853163baa9 --- /dev/null +++ b/Platforms/WASI/_build.py @@ -0,0 +1,413 @@ +#!/usr/bin/env python3 + +__lazy_modules__ = ["shutil", "sys", "tempfile", "tomllib"] + +import contextlib +import functools +import os +import pathlib +import shutil +import subprocess +import sys +import sysconfig +import tempfile +import tomllib + +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count + + +CHECKOUT = HERE = pathlib.Path(__file__).parent + +while CHECKOUT != CHECKOUT.parent: + if (CHECKOUT / "configure").is_file(): + break + CHECKOUT = CHECKOUT.parent +else: + raise FileNotFoundError( + "Unable to find the root of the CPython checkout by looking for 'configure'" + ) + +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +# Build platform can also be found via `config.guess`. +BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") + +LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" +LOCAL_SETUP_MARKER = ( + b"# Generated by Platforms/WASI .\n" + b"# Required to statically build extension modules." +) + +WASMTIME_VAR_NAME = "WASMTIME" +WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" + + +def separator(): + """Print a separator line across the terminal width.""" + try: + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) + except subprocess.CalledProcessError: + terminal_width = 80 + else: + terminal_width = int(tput_output.strip()) + print("โฏ" * terminal_width) + + +def log(emoji, message, *, spacing=None): + """Print a notification with an emoji. + + If 'spacing' is None, calculate the spacing based on the number of code points + in the emoji as terminals "eat" a space when the emoji has multiple code points. + """ + if spacing is None: + spacing = " " if len(emoji) == 1 else " " + print("".join([emoji, spacing, message])) + + +def updated_env(updates={}): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + env_vars = [ + f"\n {key}={item}" for key, item in sorted(env_diff.items()) + ] + log("๐", f"Environment changes:{''.join(env_vars)}") + + return environment + + +def subdir(working_dir, *, clean_ok=False): + """Decorator to change to a working directory.""" + + def decorator(func): + @functools.wraps(func) + def wrapper(context): + nonlocal working_dir + + if callable(working_dir): + working_dir = working_dir(context) + separator() + log("๐", os.fsdecode(working_dir)) + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): + log("๐ฎ", "Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, context=None, quiet=False, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + if context is not None: + quiet = context.quiet + + log("โฏ", " ".join(map(str, command)), spacing=" ") + if not quiet: + stdout = None + stderr = None + else: + if (logdir := getattr(context, "logdir", None)) is None: + logdir = pathlib.Path(tempfile.gettempdir()) + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + dir=logdir, + prefix="cpython-wasi-", + suffix=".log", + ) + stderr = subprocess.STDOUT + log("๐", f"Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_python_path(): + """The path to the build Python binary.""" + binary = BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {BUILD_DIR}" + ) + + return binary + + +def build_python_is_pydebug(): + """Find out if the build Python is a pydebug build.""" + test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" + result = subprocess.run( + [build_python_path(), "-c", test], + capture_output=True, + ) + return bool(result.returncode) + + +@subdir(BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + if LOCAL_SETUP.exists(): + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + log("๐", f"{LOCAL_SETUP} exists ...") + else: + log("โ ๏ธ", f"{LOCAL_SETUP} exists, but has unexpected contents") + else: + log("๐", f"Creating {LOCAL_SETUP} ...") + LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) + + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, context=context) + + +@subdir(BUILD_DIR) +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], context=context) + + binary = build_python_path() + cmd = [ + binary, + "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + log("๐", f"{binary} {version}") + + +def wasi_sdk(context): + """Find the path to the WASI SDK.""" + if wasi_sdk_path := context.wasi_sdk_path: + if not wasi_sdk_path.exists(): + raise ValueError( + "WASI SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk" + ) + return wasi_sdk_path + + with (HERE / "config.toml").open("rb") as file: + config = tomllib.load(file) + wasi_sdk_version = config["targets"]["wasi-sdk"] + + if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"): + wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) + if not wasi_sdk_path.exists(): + raise ValueError( + f"WASI SDK not found at $WASI_SDK_PATH ({wasi_sdk_path})" + ) + else: + opt_path = pathlib.Path("/opt") + # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team + # has said they don't plan to ever do a point release and all of their Git tags + # lack the ``.0`` suffix. + # Starting with WASI SDK 23, the tarballs went from containing a directory named + # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. + # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. + potential_sdks = [ + path + for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*") + if path.is_dir() + ] + if len(potential_sdks) == 1: + wasi_sdk_path = potential_sdks[0] + elif (default_path := opt_path / "wasi-sdk").is_dir(): + wasi_sdk_path = default_path + + # Starting with WASI SDK 25, a VERSION file is included in the root + # of the SDK directory that we can read to warn folks when they are using + # an unsupported version. + if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file(): + version_details = version_file.read_text(encoding="utf-8") + found_version = version_details.splitlines()[0] + # Make sure there's a trailing dot to avoid false positives if somehow the + # supported version is a prefix of the found version (e.g. `25` and `2567`). + if not found_version.startswith(f"{wasi_sdk_version}."): + major_version = found_version.partition(".")[0] + log( + "โ ๏ธ", + f" Found WASI SDK {major_version}, " + f"but WASI SDK {wasi_sdk_version} is the supported version", + ) + + # Cache the result. + context.wasi_sdk_path = wasi_sdk_path + return wasi_sdk_path + + +def wasi_sdk_env(context): + """Calculate environment variables for building with wasi-sdk.""" + wasi_sdk_path = wasi_sdk(context) + sysroot = wasi_sdk_path / "share" / "wasi-sysroot" + env = { + "CC": "clang", + "CPP": "clang-cpp", + "CXX": "clang++", + "AR": "llvm-ar", + "RANLIB": "ranlib", + } + + for env_var, binary_name in list(env.items()): + env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) + + if not wasi_sdk_path.name.startswith("wasi-sdk"): + for compiler in ["CC", "CPP", "CXX"]: + env[compiler] += f" --sysroot={sysroot}" + + env["PKG_CONFIG_PATH"] = "" + env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( + map( + os.fsdecode, + [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], + ) + ) + env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) + + env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) + env["WASI_SYSROOT"] = os.fsdecode(sysroot) + + env["PATH"] = os.pathsep.join([ + os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"], + ]) + + return env + + +def host_triple(context): + """Determine the target triple for the WASI host build.""" + if context.host_triple: + return context.host_triple + + with (HERE / "config.toml").open("rb") as file: + config = tomllib.load(file) + + # Cache the result. + context.host_triple = config["targets"]["host-triple"] + return context.host_triple + + +@subdir(lambda context: CROSS_BUILD_DIR / host_triple(context), clean_ok=True) +def configure_wasi_python(context, working_dir): + """Configure the WASI/host build.""" + config_site = os.fsdecode(HERE / "config.site-wasm32-wasi") + + wasi_build_dir = working_dir.relative_to(CHECKOUT) + + args = { + "WASMTIME": "wasmtime", + "ARGV0": f"/{wasi_build_dir}/python.wasm", + "CHECKOUT": os.fsdecode(CHECKOUT), + "WASMTIME_CONFIG_PATH": os.fsdecode(HERE / "wasmtime.toml"), + } + # Check dynamically for wasmtime in case it was specified manually via + # `--host-runner`. + if "{WASMTIME}" in context.host_runner: + if wasmtime := shutil.which("wasmtime"): + args["WASMTIME"] = wasmtime + else: + raise FileNotFoundError( + "wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime"; + ) + host_runner = context.host_runner.format_map(args) + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + build_python = os.fsdecode(build_python_path()) + # The path to `configure` MUST be relative, else `python.wasm` is unable + # to find the stdlib due to Python not recognizing that it's being + # executed from within a checkout. + configure = [ + os.path.relpath(CHECKOUT / "configure", working_dir), + f"--host={host_triple(context)}", + f"--build={BUILD_DIR.name}", + f"--with-build-python={build_python}", + ] + if build_python_is_pydebug(): + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call( + configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + context=context, + ) + + python_wasm = working_dir / "python.wasm" + exec_script = working_dir / "python.sh" + with exec_script.open("w", encoding="utf-8") as file: + file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') + exec_script.chmod(0o755) + log("๐", f"Created {exec_script} (--host-runner)... ") + sys.stdout.flush() + + +@subdir(lambda context: CROSS_BUILD_DIR / host_triple(context)) +def make_wasi_python(context, working_dir): + """Run `make` for the WASI/host build.""" + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + context=context, + ) + + exec_script = working_dir / "python.sh" + call([exec_script, "--version"], quiet=False) + log( + "๐", + f"Use `{exec_script.relative_to(pathlib.Path().absolute())}` " + "to run CPython w/ the WASI host specified by --host-runner", + ) + + +def clean_contents(context): + """Delete all files created by this script.""" + if CROSS_BUILD_DIR.exists(): + log("๐งน", f"Deleting {CROSS_BUILD_DIR} ...") + shutil.rmtree(CROSS_BUILD_DIR) + + if LOCAL_SETUP.exists(): + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + log("๐งน", f"Deleting generated {LOCAL_SETUP} ...") _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
