https://github.com/python/cpython/commit/6181b69970b93fc74d74c97ce6543c0d1b6a258c commit: 6181b69970b93fc74d74c97ce6543c0d1b6a258c branch: main author: Brett Cannon <[email protected]> committer: brettcannon <[email protected]> date: 2026-01-21T14:31:58-08:00 summary:
GH-143941: Move WASI-related files to Platforms/WASI (GH-143942) Along the way, leave a deprecated Tools/wasm/wasi/__main__.py behind for backwards-compatibility. Co-authored-by: Zachary Ware <[email protected]> files: A Misc/NEWS.d/next/Build/2026-01-16-14-27-53.gh-issue-143941.TiaE-3.rst A Platforms/WASI/.ruff.toml A Platforms/WASI/README.md A Platforms/WASI/__main__.py A Platforms/WASI/config.site-wasm32-wasi A Platforms/WASI/config.toml A Platforms/WASI/wasmtime.toml D Tools/wasm/wasi/config.site-wasm32-wasi D Tools/wasm/wasi/config.toml D Tools/wasm/wasi/wasmtime.toml M .github/CODEOWNERS M .github/workflows/reusable-wasi.yml M Tools/build/compute-changes.py M Tools/wasm/README.md M Tools/wasm/wasi.py M Tools/wasm/wasi/__main__.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 440d4c005c6978..6b6074be0a5728 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -176,6 +176,7 @@ Tools/wasm/config.site-wasm32-emscripten @freakboy3742 @emmatyping Tools/wasm/emscripten @freakboy3742 @emmatyping # WebAssembly (WASI) +Platforms/WASI @brettcannon @emmatyping @savannahostrowski Tools/wasm/wasi-env @brettcannon @emmatyping @savannahostrowski Tools/wasm/wasi.py @brettcannon @emmatyping @savannahostrowski Tools/wasm/wasi @brettcannon @emmatyping @savannahostrowski diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 4b03712eb1ee08..3c81f6ef82dc8c 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -47,14 +47,14 @@ jobs: - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: "Configure build Python" - run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug + run: python3 Platforms/WASI configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi make-build-python + run: python3 Platforms/WASI make-build-python - name: "Configure host" # `--with-pydebug` inferred from configure-build-python - run: python3 Tools/wasm/wasi configure-host -- --config-cache + run: python3 Platforms/WASI configure-host -- --config-cache - name: "Make host" - run: python3 Tools/wasm/wasi make-host + run: python3 Platforms/WASI make-host - name: "Display build info" run: make --directory "${CROSS_BUILD_WASI}" pythoninfo - name: "Test" diff --git a/Misc/NEWS.d/next/Build/2026-01-16-14-27-53.gh-issue-143941.TiaE-3.rst b/Misc/NEWS.d/next/Build/2026-01-16-14-27-53.gh-issue-143941.TiaE-3.rst new file mode 100644 index 00000000000000..68839364a0ddee --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-01-16-14-27-53.gh-issue-143941.TiaE-3.rst @@ -0,0 +1,3 @@ +Move WASI-related files to :file:`Platforms/WASI`. Along the way, leave a +deprecated :file:`Tools/wasm/wasi/__main__.py` behind for +backwards-compatibility. diff --git a/Platforms/WASI/.ruff.toml b/Platforms/WASI/.ruff.toml new file mode 100644 index 00000000000000..3d8e59fa3f22c4 --- /dev/null +++ b/Platforms/WASI/.ruff.toml @@ -0,0 +1,25 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] +ignore = [ + "E501", # Line too long +] diff --git a/Platforms/WASI/README.md b/Platforms/WASI/README.md new file mode 100644 index 00000000000000..62c82924d437b1 --- /dev/null +++ b/Platforms/WASI/README.md @@ -0,0 +1,59 @@ +# Python WASI (wasm32-wasi) build + +**WASI support is [tier 2](https://peps.python.org/pep-0011/#tier-2).** + +This directory contains configuration and helpers to facilitate cross +compilation of CPython to WebAssembly (WASM) using WASI. WASI builds +use WASM runtimes such as [wasmtime](https://wasmtime.dev/). + +**NOTE**: If you are looking for general information about WebAssembly that is +not directly related to CPython, please see https://github.com/psf/webassembly. + +## Build + +See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi). + +## Detecting WASI builds + +### Python code + +```python +import os, sys + +if sys.platform == "wasi": + # Python on WASI + ... + +if os.name == "posix": + # WASM platforms identify as POSIX-like. + # Windows does not provide os.uname(). + machine = os.uname().machine + if machine.startswith("wasm"): + # WebAssembly (wasm32, wasm64 potentially in the future) +``` + +```python +>>> import os, sys +>>> os.uname() +posix.uname_result( + sysname='wasi', + nodename='(none)', + release='0.0.0', + version='0.0.0', + machine='wasm32' +) +>>> os.name +'posix' +>>> sys.platform +'wasi' +``` + +### C code + +WASI SDK defines several built-in macros. You can dump a full list of built-ins +with ``/path/to/wasi-sdk/bin/clang -dM -E - < /dev/null``. + +* WebAssembly ``__wasm__`` (also ``__wasm``) +* wasm32 ``__wasm32__`` (also ``__wasm32``) +* wasm64 ``__wasm64__`` +* WASI ``__wasi__`` diff --git a/Platforms/WASI/__main__.py b/Platforms/WASI/__main__.py new file mode 100644 index 00000000000000..8302432fd2f106 --- /dev/null +++ b/Platforms/WASI/__main__.py @@ -0,0 +1,557 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import functools +import os + +import tomllib + +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count +import pathlib +import shutil +import subprocess +import sys +import sysconfig +import tempfile + +CHECKOUT = HERE = pathlib.Path(__file__).parent + +while CHECKOUT != CHECKOUT.parent: + if (CHECKOUT / "configure").is_file(): + break + CHECKOUT = CHECKOUT.parent +else: + raise FileNotFoundError( + "Unable to find the root of the CPython checkout by looking for 'configure'" + ) + +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +# Build platform can also be found via `config.guess`. +BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") + +LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" +LOCAL_SETUP_MARKER = ( + b"# Generated by Platforms/WASI .\n" + b"# Required to statically build extension modules." +) + +WASI_SDK_VERSION = 29 + +WASMTIME_VAR_NAME = "WASMTIME" +WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" + + +def separator(): + """Print a separator line across the terminal width.""" + try: + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) + except subprocess.CalledProcessError: + terminal_width = 80 + else: + terminal_width = int(tput_output.strip()) + print("โฏ" * terminal_width) + + +def log(emoji, message, *, spacing=None): + """Print a notification with an emoji. + + If 'spacing' is None, calculate the spacing based on the number of code points + in the emoji as terminals "eat" a space when the emoji has multiple code points. + """ + if spacing is None: + spacing = " " if len(emoji) == 1 else " " + print("".join([emoji, spacing, message])) + + +def updated_env(updates={}): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + env_vars = ( + f"\n {key}={item}" for key, item in sorted(env_diff.items()) + ) + log("๐", f"Environment changes:{''.join(env_vars)}") + + return environment + + +def subdir(working_dir, *, clean_ok=False): + """Decorator to change to a working directory.""" + + def decorator(func): + @functools.wraps(func) + def wrapper(context): + nonlocal working_dir + + if callable(working_dir): + working_dir = working_dir(context) + separator() + log("๐", os.fsdecode(working_dir)) + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): + log("๐ฎ", "Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, context=None, quiet=False, logdir=None, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + if context is not None: + quiet = context.quiet + logdir = context.logdir + elif quiet and logdir is None: + raise ValueError("When quiet is True, logdir must be specified") + + log("โฏ", " ".join(map(str, command)), spacing=" ") + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + dir=logdir, + prefix="cpython-wasi-", + suffix=".log", + ) + stderr = subprocess.STDOUT + log("๐", f"Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_python_path(): + """The path to the build Python binary.""" + binary = BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {BUILD_DIR}" + ) + + return binary + + +def build_python_is_pydebug(): + """Find out if the build Python is a pydebug build.""" + test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" + result = subprocess.run( + [build_python_path(), "-c", test], + capture_output=True, + ) + return bool(result.returncode) + + +@subdir(BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + if LOCAL_SETUP.exists(): + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + log("๐", f"{LOCAL_SETUP} exists ...") + else: + log("โ ๏ธ", f"{LOCAL_SETUP} exists, but has unexpected contents") + else: + log("๐", f"Creating {LOCAL_SETUP} ...") + LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) + + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, context=context) + + +@subdir(BUILD_DIR) +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], context=context) + + binary = build_python_path() + cmd = [ + binary, + "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + log("๐", f"{binary} {version}") + + +def find_wasi_sdk(config): + """Find the path to the WASI SDK.""" + wasi_sdk_path = None + wasi_sdk_version = config["targets"]["wasi-sdk"] + + if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"): + wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) + else: + opt_path = pathlib.Path("/opt") + # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team + # has said they don't plan to ever do a point release and all of their Git tags + # lack the ``.0`` suffix. + # Starting with WASI SDK 23, the tarballs went from containing a directory named + # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. + # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. + potential_sdks = [ + path + for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*") + if path.is_dir() + ] + if len(potential_sdks) == 1: + wasi_sdk_path = potential_sdks[0] + elif (default_path := opt_path / "wasi-sdk").is_dir(): + wasi_sdk_path = default_path + + # Starting with WASI SDK 25, a VERSION file is included in the root + # of the SDK directory that we can read to warn folks when they are using + # an unsupported version. + if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file(): + version_details = version_file.read_text(encoding="utf-8") + found_version = version_details.splitlines()[0] + # Make sure there's a trailing dot to avoid false positives if somehow the + # supported version is a prefix of the found version (e.g. `25` and `2567`). + if not found_version.startswith(f"{wasi_sdk_version}."): + major_version = found_version.partition(".")[0] + log( + "โ ๏ธ", + f" Found WASI SDK {major_version}, " + f"but WASI SDK {wasi_sdk_version} is the supported version", + ) + + return wasi_sdk_path + + +def wasi_sdk_env(context): + """Calculate environment variables for building with wasi-sdk.""" + wasi_sdk_path = context.wasi_sdk_path + sysroot = wasi_sdk_path / "share" / "wasi-sysroot" + env = { + "CC": "clang", + "CPP": "clang-cpp", + "CXX": "clang++", + "AR": "llvm-ar", + "RANLIB": "ranlib", + } + + for env_var, binary_name in list(env.items()): + env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) + + if not wasi_sdk_path.name.startswith("wasi-sdk"): + for compiler in ["CC", "CPP", "CXX"]: + env[compiler] += f" --sysroot={sysroot}" + + env["PKG_CONFIG_PATH"] = "" + env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( + map( + os.fsdecode, + [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], + ) + ) + env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) + + env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) + env["WASI_SYSROOT"] = os.fsdecode(sysroot) + + env["PATH"] = os.pathsep.join([ + os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"], + ]) + + return env + + +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) +def configure_wasi_python(context, working_dir): + """Configure the WASI/host build.""" + if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): + raise ValueError( + "WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk" + ) + + config_site = os.fsdecode(HERE / "config.site-wasm32-wasi") + + wasi_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = BUILD_DIR / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) + lib_dir = os.fsdecode(lib_dirs[0]) + python_version = lib_dir.rpartition("-")[-1] + sysconfig_data_dir = ( + f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + ) + + # Use PYTHONPATH to include sysconfig data which must be anchored to the + # WASI guest's `/` directory. + args = { + "PYTHONPATH": f"/{sysconfig_data_dir}", + "PYTHON_WASM": working_dir / "python.wasm", + } + # Check dynamically for wasmtime in case it was specified manually via + # `--host-runner`. + if WASMTIME_HOST_RUNNER_VAR in context.host_runner: + if wasmtime := shutil.which("wasmtime"): + args[WASMTIME_VAR_NAME] = wasmtime + else: + raise FileNotFoundError( + "wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime"; + ) + host_runner = context.host_runner.format_map(args) + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + build_python = os.fsdecode(build_python_path()) + # The path to `configure` MUST be relative, else `python.wasm` is unable + # to find the stdlib due to Python not recognizing that it's being + # executed from within a checkout. + configure = [ + os.path.relpath(CHECKOUT / "configure", working_dir), + f"--host={context.host_triple}", + f"--build={BUILD_DIR.name}", + f"--with-build-python={build_python}", + ] + if build_python_is_pydebug(): + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call( + configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + context=context, + ) + + python_wasm = working_dir / "python.wasm" + exec_script = working_dir / "python.sh" + with exec_script.open("w", encoding="utf-8") as file: + file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') + exec_script.chmod(0o755) + log("๐", f"Created {exec_script} (--host-runner)... ") + sys.stdout.flush() + + +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) +def make_wasi_python(context, working_dir): + """Run `make` for the WASI/host build.""" + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + context=context, + ) + + exec_script = working_dir / "python.sh" + call([exec_script, "--version"], quiet=False) + log( + "๐", + f"Use `{exec_script.relative_to(context.init_dir)}` " + "to run CPython w/ the WASI host specified by --host-runner", + ) + + +def clean_contents(context): + """Delete all files created by this script.""" + if CROSS_BUILD_DIR.exists(): + log("๐งน", f"Deleting {CROSS_BUILD_DIR} ...") + shutil.rmtree(CROSS_BUILD_DIR) + + if LOCAL_SETUP.exists(): + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + log("๐งน", f"Deleting generated {LOCAL_SETUP} ...") + + +def build_steps(*steps): + """Construct a command from other steps.""" + + def builder(context): + for step in steps: + step(context) + + return builder + + +def main(): + with (HERE / "config.toml").open("rb") as file: + config = tomllib.load(file) + default_wasi_sdk = find_wasi_sdk(config) + default_host_triple = config["targets"]["host-triple"] + default_host_runner = ( + f"{WASMTIME_HOST_RUNNER_VAR} run " + # For setting PYTHONPATH to the sysconfig data directory. + "--env PYTHONPATH={PYTHONPATH} " + # Map the checkout to / to load the stdlib from /Lib. + f"--dir {os.fsdecode(CHECKOUT)}::/ " + # Flags involving --optimize, --codegen, --debug, --wasm, and --wasi can be kept + # in a config file. + # We are using such a file to act as defaults in case a user wants to override + # only some of the settings themselves, make it easy to modify settings + # post-build so that they immediately apply to the Makefile instead of having to + # regenerate it, and allow for easy copying of the settings for anyone else who + # may want to use them. + f"--config {os.fsdecode(HERE / 'wasmtime.toml')}" + ) + default_logdir = pathlib.Path(tempfile.gettempdir()) + + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the build Python" + ) + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + build_python = subcommands.add_parser( + "build-python", help="Build the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)", + ) + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/WASI" + ) + build_host = subcommands.add_parser( + "build-host", help="Build the host/WASI Python" + ) + subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + for subcommand in ( + build, + configure_build, + make_build, + build_python, + configure_host, + make_host, + build_host, + ): + subcommand.add_argument( + "--quiet", + action="store_true", + default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file", + ) + subcommand.add_argument( + "--logdir", + type=pathlib.Path, + default=default_logdir, + help=f"Directory to store log files; defaults to {default_logdir}", + ) + for subcommand in ( + configure_build, + configure_host, + build_python, + build_host, + ): + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) + for subcommand in ( + build, + configure_build, + configure_host, + build_python, + build_host, + ): + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + for subcommand in build, configure_host, build_host: + subcommand.add_argument( + "--wasi-sdk", + type=pathlib.Path, + dest="wasi_sdk_path", + default=default_wasi_sdk, + help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}", + ) + subcommand.add_argument( + "--host-runner", + action="store", + default=default_host_runner, + dest="host_runner", + help="Command template for running the WASI host; defaults to " + f"`{default_host_runner}`", + ) + for subcommand in build, configure_host, make_host, build_host: + subcommand.add_argument( + "--host-triple", + action="store", + default=default_host_triple, + help="The target triple for the WASI host build; " + f"defaults to {default_host_triple}", + ) + + context = parser.parse_args() + context.init_dir = pathlib.Path().absolute() + + build_build_python = build_steps(configure_build_python, make_build_python) + build_wasi_python = build_steps(configure_wasi_python, make_wasi_python) + + dispatch = { + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "build-python": build_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build-host": build_wasi_python, + "build": build_steps(build_build_python, build_wasi_python), + "clean": clean_contents, + } + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/Tools/wasm/wasi/config.site-wasm32-wasi b/Platforms/WASI/config.site-wasm32-wasi similarity index 100% rename from Tools/wasm/wasi/config.site-wasm32-wasi rename to Platforms/WASI/config.site-wasm32-wasi diff --git a/Tools/wasm/wasi/config.toml b/Platforms/WASI/config.toml similarity index 100% rename from Tools/wasm/wasi/config.toml rename to Platforms/WASI/config.toml diff --git a/Tools/wasm/wasi/wasmtime.toml b/Platforms/WASI/wasmtime.toml similarity index 100% rename from Tools/wasm/wasi/wasmtime.toml rename to Platforms/WASI/wasmtime.toml diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 88b7856b9c736a..08aba5ef5423fe 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -50,7 +50,7 @@ ANDROID_DIRS = frozenset({"Android"}) IOS_DIRS = frozenset({"Apple", "iOS"}) MACOS_DIRS = frozenset({"Mac"}) -WASI_DIRS = frozenset({Path("Tools", "wasm")}) +WASI_DIRS = frozenset({Path("Platforms", "WASI")}) LIBRARY_FUZZER_PATHS = frozenset({ # All C/CPP fuzzers. @@ -194,7 +194,7 @@ def get_file_platform(file: Path) -> str | None: return "ios" if first_part in ANDROID_DIRS: return "android" - if len(file.parts) >= 2 and Path(*file.parts[:2]) in WASI_DIRS: # Tools/wasm/ + if len(file.parts) >= 2 and Path(*file.parts[:2]) in WASI_DIRS: return "wasi" return None diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 35685db07b1431..46228a5212a315 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -1,19 +1,16 @@ -# Python WebAssembly (WASM) build +# Python Emscripten (wasm32-emscripten) build -**WASI support is [tier 2](https://peps.python.org/pep-0011/#tier-2).** **Emscripten support is [tier 3](https://peps.python.org/pep-0011/#tier-3).** This directory contains configuration and helpers to facilitate cross -compilation of CPython to WebAssembly (WASM). Python supports Emscripten -(*wasm32-emscripten*) and WASI (*wasm32-wasi*) targets. Emscripten builds -run in modern browsers and JavaScript runtimes like *Node.js*. WASI builds -use WASM runtimes such as *wasmtime*. +compilation of CPython to WebAssembly (WASM) using Emscripten. Emscripten builds +run in modern browsers and JavaScript runtimes like *Node.js*. + +For WASI builds, see [Platforms/WASI](../../Platforms/WASI/README.md). **NOTE**: If you are looking for general information about WebAssembly that is not directly related to CPython, please see https://github.com/psf/webassembly. -## Emscripten (wasm32-emscripten) - ### Build See [the devguide instructions for building for Emscripten](https://devguide.python.org/getting-started/setup-building/#emscripten). @@ -192,11 +189,7 @@ await createEmscriptenModule({ - Test modules are disabled by default. Use ``--enable-test-modules`` build test modules like ``_testcapi``. -## WASI (wasm32-wasi) - -See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi). - -## Detecting WebAssembly builds +## Detecting Emscripten builds ### Python code @@ -206,9 +199,6 @@ import os, sys if sys.platform == "emscripten": # Python on Emscripten ... -if sys.platform == "wasi": - # Python on WASI - ... if os.name == "posix": # WASM platforms identify as POSIX-like. @@ -251,31 +241,14 @@ sys._emscripten_info( ) ``` -```python ->>> import os, sys ->>> os.uname() -posix.uname_result( - sysname='wasi', - nodename='(none)', - release='0.0.0', - version='0.0.0', - machine='wasm32' -) ->>> os.name -'posix' ->>> sys.platform -'wasi' -``` - ### C code -Emscripten SDK and WASI SDK define several built-in macros. You can dump a -full list of built-ins with ``emcc -dM -E - < /dev/null`` and -``/path/to/wasi-sdk/bin/clang -dM -E - < /dev/null``. +Emscripten SDK defines several built-in macros. You can dump a full list of +built-ins with ``emcc -dM -E - < /dev/null``. * WebAssembly ``__wasm__`` (also ``__wasm``) * wasm32 ``__wasm32__`` (also ``__wasm32``) * wasm64 ``__wasm64__`` * Emscripten ``__EMSCRIPTEN__`` (also ``EMSCRIPTEN``) * Emscripten version ``__EMSCRIPTEN_major__``, ``__EMSCRIPTEN_minor__``, ``__EMSCRIPTEN_tiny__`` -* WASI ``__wasi__`` + diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index af55e03d10f754..72a708c45c02af 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -1,12 +1,5 @@ if __name__ == "__main__": import pathlib import runpy - import sys - - print( - "โ ๏ธ WARNING: This script is deprecated and slated for removal in Python 3.20; " - "execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n", - file=sys.stderr, - ) runpy.run_path(pathlib.Path(__file__).parent / "wasi", run_name="__main__") diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index 3d9a2472d4dcbf..e6126eddde1f84 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -1,554 +1,14 @@ -#!/usr/bin/env python3 - -import argparse -import contextlib -import functools -import os - -import tomllib - -try: - from os import process_cpu_count as cpu_count -except ImportError: - from os import cpu_count -import pathlib -import shutil -import subprocess -import sys -import sysconfig -import tempfile - -HERE = pathlib.Path(__file__).parent - -# Path is: cpython/Tools/wasm/wasi -CHECKOUT = HERE.parent.parent.parent -assert (CHECKOUT / "configure").is_file(), ( - "Please update the location of the file" -) - -CROSS_BUILD_DIR = CHECKOUT / "cross-build" -# Build platform can also be found via `config.guess`. -BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") - -LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = ( - b"# Generated by Tools/wasm/wasi .\n" - b"# Required to statically build extension modules." -) - -WASI_SDK_VERSION = 29 - -WASMTIME_VAR_NAME = "WASMTIME" -WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" - - -def separator(): - """Print a separator line across the terminal width.""" - try: - tput_output = subprocess.check_output( - ["tput", "cols"], encoding="utf-8" - ) - except subprocess.CalledProcessError: - terminal_width = 80 - else: - terminal_width = int(tput_output.strip()) - print("โฏ" * terminal_width) - - -def log(emoji, message, *, spacing=None): - """Print a notification with an emoji. - - If 'spacing' is None, calculate the spacing based on the number of code points - in the emoji as terminals "eat" a space when the emoji has multiple code points. - """ - if spacing is None: - spacing = " " if len(emoji) == 1 else " " - print("".join([emoji, spacing, message])) - - -def updated_env(updates={}): - """Create a new dict representing the environment to use. - - The changes made to the execution environment are printed out. - """ - env_defaults = {} - # https://reproducible-builds.org/docs/source-date-epoch/ - git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] - try: - epoch = subprocess.check_output( - git_epoch_cmd, encoding="utf-8" - ).strip() - env_defaults["SOURCE_DATE_EPOCH"] = epoch - except subprocess.CalledProcessError: - pass # Might be building from a tarball. - # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. - environment = env_defaults | os.environ | updates - - env_diff = {} - for key, value in environment.items(): - if os.environ.get(key) != value: - env_diff[key] = value - - env_vars = ( - f"\n {key}={item}" for key, item in sorted(env_diff.items()) - ) - log("๐", f"Environment changes:{''.join(env_vars)}") - - return environment - - -def subdir(working_dir, *, clean_ok=False): - """Decorator to change to a working directory.""" - - def decorator(func): - @functools.wraps(func) - def wrapper(context): - nonlocal working_dir - - if callable(working_dir): - working_dir = working_dir(context) - separator() - log("๐", os.fsdecode(working_dir)) - if ( - clean_ok - and getattr(context, "clean", False) - and working_dir.exists() - ): - log("๐ฎ", "Deleting directory (--clean)...") - shutil.rmtree(working_dir) - - working_dir.mkdir(parents=True, exist_ok=True) - - with contextlib.chdir(working_dir): - return func(context, working_dir) - - return wrapper - - return decorator - - -def call(command, *, context=None, quiet=False, logdir=None, **kwargs): - """Execute a command. - - If 'quiet' is true, then redirect stdout and stderr to a temporary file. - """ - if context is not None: - quiet = context.quiet - logdir = context.logdir - elif quiet and logdir is None: - raise ValueError("When quiet is True, logdir must be specified") - - log("โฏ", " ".join(map(str, command)), spacing=" ") - if not quiet: - stdout = None - stderr = None - else: - stdout = tempfile.NamedTemporaryFile( - "w", - encoding="utf-8", - delete=False, - dir=logdir, - prefix="cpython-wasi-", - suffix=".log", - ) - stderr = subprocess.STDOUT - log("๐", f"Logging output to {stdout.name} (--quiet)...") - - subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) - - -def build_python_path(): - """The path to the build Python binary.""" - binary = BUILD_DIR / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError( - f"Unable to find `python(.exe)` in {BUILD_DIR}" - ) - - return binary - - -def build_python_is_pydebug(): - """Find out if the build Python is a pydebug build.""" - test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" - result = subprocess.run( - [build_python_path(), "-c", test], - capture_output=True, - ) - return bool(result.returncode) - - -@subdir(BUILD_DIR, clean_ok=True) -def configure_build_python(context, working_dir): - """Configure the build/host Python.""" - if LOCAL_SETUP.exists(): - if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: - log("๐", f"{LOCAL_SETUP} exists ...") - else: - log("โ ๏ธ", f"{LOCAL_SETUP} exists, but has unexpected contents") - else: - log("๐", f"Creating {LOCAL_SETUP} ...") - LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) - - configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] - if context.args: - configure.extend(context.args) - - call(configure, context=context) - - -@subdir(BUILD_DIR) -def make_build_python(context, working_dir): - """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], context=context) - - binary = build_python_path() - cmd = [ - binary, - "-c", - "import sys; " - "print(f'{sys.version_info.major}.{sys.version_info.minor}')", - ] - version = subprocess.check_output(cmd, encoding="utf-8").strip() - - log("๐", f"{binary} {version}") - - -def find_wasi_sdk(config): - """Find the path to the WASI SDK.""" - wasi_sdk_path = None - wasi_sdk_version = config["targets"]["wasi-sdk"] - - if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"): - wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) - else: - opt_path = pathlib.Path("/opt") - # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team - # has said they don't plan to ever do a point release and all of their Git tags - # lack the ``.0`` suffix. - # Starting with WASI SDK 23, the tarballs went from containing a directory named - # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. - # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. - potential_sdks = [ - path - for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*") - if path.is_dir() - ] - if len(potential_sdks) == 1: - wasi_sdk_path = potential_sdks[0] - elif (default_path := opt_path / "wasi-sdk").is_dir(): - wasi_sdk_path = default_path - - # Starting with WASI SDK 25, a VERSION file is included in the root - # of the SDK directory that we can read to warn folks when they are using - # an unsupported version. - if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file(): - version_details = version_file.read_text(encoding="utf-8") - found_version = version_details.splitlines()[0] - # Make sure there's a trailing dot to avoid false positives if somehow the - # supported version is a prefix of the found version (e.g. `25` and `2567`). - if not found_version.startswith(f"{wasi_sdk_version}."): - major_version = found_version.partition(".")[0] - log( - "โ ๏ธ", - f" Found WASI SDK {major_version}, " - f"but WASI SDK {wasi_sdk_version} is the supported version", - ) - - return wasi_sdk_path - - -def wasi_sdk_env(context): - """Calculate environment variables for building with wasi-sdk.""" - wasi_sdk_path = context.wasi_sdk_path - sysroot = wasi_sdk_path / "share" / "wasi-sysroot" - env = { - "CC": "clang", - "CPP": "clang-cpp", - "CXX": "clang++", - "AR": "llvm-ar", - "RANLIB": "ranlib", - } - - for env_var, binary_name in list(env.items()): - env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) - - if not wasi_sdk_path.name.startswith("wasi-sdk"): - for compiler in ["CC", "CPP", "CXX"]: - env[compiler] += f" --sysroot={sysroot}" - - env["PKG_CONFIG_PATH"] = "" - env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( - map( - os.fsdecode, - [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], - ) - ) - env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) - - env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) - env["WASI_SYSROOT"] = os.fsdecode(sysroot) - - env["PATH"] = os.pathsep.join([ - os.fsdecode(wasi_sdk_path / "bin"), - os.environ["PATH"], - ]) - - return env - - -@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) -def configure_wasi_python(context, working_dir): - """Configure the WASI/host build.""" - if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError( - "WASI-SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk" - ) - - config_site = os.fsdecode(HERE / "config.site-wasm32-wasi") - - wasi_build_dir = working_dir.relative_to(CHECKOUT) - - python_build_dir = BUILD_DIR / "build" - lib_dirs = list(python_build_dir.glob("lib.*")) - assert len(lib_dirs) == 1, ( - f"Expected a single lib.* directory in {python_build_dir}" - ) - lib_dir = os.fsdecode(lib_dirs[0]) - python_version = lib_dir.rpartition("-")[-1] - sysconfig_data_dir = ( - f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" - ) - - # Use PYTHONPATH to include sysconfig data which must be anchored to the - # WASI guest's `/` directory. - args = { - "PYTHONPATH": f"/{sysconfig_data_dir}", - "PYTHON_WASM": working_dir / "python.wasm", - } - # Check dynamically for wasmtime in case it was specified manually via - # `--host-runner`. - if WASMTIME_HOST_RUNNER_VAR in context.host_runner: - if wasmtime := shutil.which("wasmtime"): - args[WASMTIME_VAR_NAME] = wasmtime - else: - raise FileNotFoundError( - "wasmtime not found; download from " - "https://github.com/bytecodealliance/wasmtime"; - ) - host_runner = context.host_runner.format_map(args) - env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} - build_python = os.fsdecode(build_python_path()) - # The path to `configure` MUST be relative, else `python.wasm` is unable - # to find the stdlib due to Python not recognizing that it's being - # executed from within a checkout. - configure = [ - os.path.relpath(CHECKOUT / "configure", working_dir), - f"--host={context.host_triple}", - f"--build={BUILD_DIR.name}", - f"--with-build-python={build_python}", - ] - if build_python_is_pydebug(): - configure.append("--with-pydebug") - if context.args: - configure.extend(context.args) - call( - configure, - env=updated_env(env_additions | wasi_sdk_env(context)), - context=context, - ) - - python_wasm = working_dir / "python.wasm" - exec_script = working_dir / "python.sh" - with exec_script.open("w", encoding="utf-8") as file: - file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') - exec_script.chmod(0o755) - log("๐", f"Created {exec_script} (--host-runner)... ") - sys.stdout.flush() - - -@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) -def make_wasi_python(context, working_dir): - """Run `make` for the WASI/host build.""" - call( - ["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - context=context, - ) - - exec_script = working_dir / "python.sh" - call([exec_script, "--version"], quiet=False) - log( - "๐", - f"Use `{exec_script.relative_to(context.init_dir)}` " - "to run CPython w/ the WASI host specified by --host-runner", - ) - - -def clean_contents(context): - """Delete all files created by this script.""" - if CROSS_BUILD_DIR.exists(): - log("๐งน", f"Deleting {CROSS_BUILD_DIR} ...") - shutil.rmtree(CROSS_BUILD_DIR) - - if LOCAL_SETUP.exists(): - if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: - log("๐งน", f"Deleting generated {LOCAL_SETUP} ...") - - -def build_steps(*steps): - """Construct a command from other steps.""" - - def builder(context): - for step in steps: - step(context) - - return builder - - -def main(): - with (HERE / "config.toml").open("rb") as file: - config = tomllib.load(file) - default_wasi_sdk = find_wasi_sdk(config) - default_host_triple = config["targets"]["host-triple"] - default_host_runner = ( - f"{WASMTIME_HOST_RUNNER_VAR} run " - # For setting PYTHONPATH to the sysconfig data directory. - "--env PYTHONPATH={PYTHONPATH} " - # Map the checkout to / to load the stdlib from /Lib. - f"--dir {os.fsdecode(CHECKOUT)}::/ " - # Flags involving --optimize, --codegen, --debug, --wasm, and --wasi can be kept - # in a config file. - # We are using such a file to act as defaults in case a user wants to override - # only some of the settings themselves, make it easy to modify settings - # post-build so that they immediately apply to the Makefile instead of having to - # regenerate it, and allow for easy copying of the settings for anyone else who - # may want to use them. - f"--config {os.fsdecode(HERE / 'wasmtime.toml')}" - ) - default_logdir = pathlib.Path(tempfile.gettempdir()) +if __name__ == "__main__": + import pathlib + import runpy + import sys - parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser( - "configure-build-python", help="Run `configure` for the build Python" - ) - make_build = subcommands.add_parser( - "make-build-python", help="Run `make` for the build Python" - ) - build_python = subcommands.add_parser( - "build-python", help="Build the build Python" - ) - configure_host = subcommands.add_parser( - "configure-host", - help="Run `configure` for the " - "host/WASI (pydebug builds " - "are inferred from the build " - "Python)", - ) - make_host = subcommands.add_parser( - "make-host", help="Run `make` for the host/WASI" + print( + "โ ๏ธ WARNING: This script is deprecated and slated for removal in Python 3.20; " + "execute the `Platforms/WASI/` directory instead (i.e. `python Platforms/WASI`)\n", + file=sys.stderr, ) - build_host = subcommands.add_parser( - "build-host", help="Build the host/WASI Python" - ) - subcommands.add_parser( - "clean", help="Delete files and directories created by this script" - ) - for subcommand in ( - build, - configure_build, - make_build, - build_python, - configure_host, - make_host, - build_host, - ): - subcommand.add_argument( - "--quiet", - action="store_true", - default=False, - dest="quiet", - help="Redirect output from subprocesses to a log file", - ) - subcommand.add_argument( - "--logdir", - type=pathlib.Path, - default=default_logdir, - help=f"Directory to store log files; defaults to {default_logdir}", - ) - for subcommand in ( - configure_build, - configure_host, - build_python, - build_host, - ): - subcommand.add_argument( - "--clean", - action="store_true", - default=False, - dest="clean", - help="Delete any relevant directories before building", - ) - for subcommand in ( - build, - configure_build, - configure_host, - build_python, - build_host, - ): - subcommand.add_argument( - "args", nargs="*", help="Extra arguments to pass to `configure`" - ) - for subcommand in build, configure_host, build_host: - subcommand.add_argument( - "--wasi-sdk", - type=pathlib.Path, - dest="wasi_sdk_path", - default=default_wasi_sdk, - help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}", - ) - subcommand.add_argument( - "--host-runner", - action="store", - default=default_host_runner, - dest="host_runner", - help="Command template for running the WASI host; defaults to " - f"`{default_host_runner}`", - ) - for subcommand in build, configure_host, make_host, build_host: - subcommand.add_argument( - "--host-triple", - action="store", - default=default_host_triple, - help="The target triple for the WASI host build; " - f"defaults to {default_host_triple}", - ) - - context = parser.parse_args() - context.init_dir = pathlib.Path().absolute() - build_build_python = build_steps(configure_build_python, make_build_python) - build_wasi_python = build_steps(configure_wasi_python, make_wasi_python) + checkout = pathlib.Path(__file__).parent.parent.parent.parent - dispatch = { - "configure-build-python": configure_build_python, - "make-build-python": make_build_python, - "build-python": build_build_python, - "configure-host": configure_wasi_python, - "make-host": make_wasi_python, - "build-host": build_wasi_python, - "build": build_steps(build_build_python, build_wasi_python), - "clean": clean_contents, - } - dispatch[context.subcommand](context) - - -if __name__ == "__main__": - main() + runpy.run_path(checkout / "Platforms" / "WASI", run_name="__main__") _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
