https://github.com/python/cpython/commit/d6b77c8686953a4e87348843c312f9f35d2cd033 commit: d6b77c8686953a4e87348843c312f9f35d2cd033 branch: main author: Clément Péron <[email protected]> committer: freakboy3742 <[email protected]> date: 2026-10-22T19:58:32Z summary:
gh-156780: Emscripten: support building CPython with static linking (#156781) Adds the ability to build Emscripten statically, so that it can be linked in as an embedded library. Includes a test of the embedding case. --------- Co-authored-by: Claude Fable 5.1 <[email protected]> Co-authored-by: Russell Keith-Magee <[email protected]> files: A Misc/NEWS.d/next/Build/2026-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst A Platforms/emscripten/web_embed_test/main.c A Platforms/emscripten/web_embed_test/run_test.sh A Programs/emscripten_beforemain.c M .github/workflows/reusable-emscripten.yml M Makefile.pre.in M Modules/_testinternalcapi.c M Python/emscripten_syscalls.c M configure M configure.ac diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index 18afdb5278a766e..f4fb5d4a5b0b6ab 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -79,3 +79,5 @@ jobs: run: python3 Platforms/emscripten run --test - name: "Test Repl" run: Platforms/emscripten/browser_test/run_test.sh + - name: "Test embedding" + run: Platforms/emscripten/web_embed_test/run_test.sh diff --git a/Makefile.pre.in b/Makefile.pre.in index cae9cae9d779bea..3d971ddc8cd1326 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -971,9 +971,12 @@ clinic: check-clean-src clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py -f $(srcdir)/Lib/test/clinic.test.c +# Objects linked into the interpreter only, not into libpython. +PLATFORM_MAIN_OBJS= @PLATFORM_MAIN_OBJS@ + # Build the interpreter -$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) - $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) +$(BUILDPYTHON): Programs/python.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_DEPS) + $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -1120,6 +1123,20 @@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) cp python.mjs web_example/python.mjs cp python.wasm web_example/python.wasm +WEB_EMBED_TEST_DIR=$(EMSCRIPTEN_DIR)/web_embed_test + +# Linked like an embedder would: against libpython.a, without $(LINKFORSHARED) +# and so without -sMAIN_MODULE or the interpreter's -sEXPORTED_FUNCTIONS. +web_embed_test/main.js: $(WEB_EMBED_TEST_DIR)/main.c $(LIBRARY) $(ZIP_STDLIB) + @mkdir -p web_embed_test + $(LINKCC) $(PY_STDMODULE_CFLAGS) $(PY_CORE_EXE_LDFLAGS) -O2 -g0 -o $@ \ + $(WEB_EMBED_TEST_DIR)/main.c $(LIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \ + -sALLOW_MEMORY_GROWTH -sSTACK_SIZE=5MB \ + --preload-file $(ZIP_STDLIB)@/lib/stdlib.zip + +.PHONY: web_embed_test +web_embed_test: web_embed_test/main.js + .PHONY: web_example web_example: web_example/python.mjs web_example/python.worker.mjs web_example/index.html web_example/server.py web_example/$(ZIP_STDLIB) @@ -1620,8 +1637,8 @@ regen-re: $(BUILDPYTHON) # using Tools/build/generate_re_casefix.py $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/generate_re_casefix.py $(srcdir)/Lib/re/_casefix.py -Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) - $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) +Programs/_testembed: Programs/_testembed.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_DEPS) + $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) ############################################################################ # "Bootstrap Python" used to run Programs/_freeze_module.py @@ -2251,7 +2268,7 @@ regen-slots: Python/slots.toml $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_slots.py \ --generate-all -$(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) +$(LIBRARY_OBJS) $(MODOBJS) Programs/python.o $(PLATFORM_MAIN_OBJS): $(PYTHON_HEADERS) ###################################################################### @@ -2977,6 +2994,9 @@ libainstall: all scripts fi; \ fi; \ $(INSTALL_DATA) Programs/python.o $(DESTDIR)$(LIBPL)/python.o; \ + for i in $(PLATFORM_MAIN_OBJS); do \ + $(INSTALL_DATA) $$i $(DESTDIR)$(LIBPL); \ + done; \ fi $(INSTALL_DATA) Modules/config.c $(DESTDIR)$(LIBPL)/config.c $(INSTALL_DATA) $(srcdir)/Modules/config.c.in $(DESTDIR)$(LIBPL)/config.c.in @@ -3291,7 +3311,7 @@ clean-retain-profile: pycremoval find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt -rm -f _bootstrap_python - -rm -rf web_example python.mjs python.wasm python*.symbols python*.map + -rm -rf web_example web_embed_test python.mjs python.wasm python*.symbols python*.map -rm -f Programs/_testembed Programs/_freeze_module -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h diff --git a/Misc/NEWS.d/next/Build/2026-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst b/Misc/NEWS.d/next/Build/2026-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst new file mode 100644 index 000000000000000..fe664df69097405 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst @@ -0,0 +1 @@ +Emscripten ``libpython.a`` can now be linked into a program statically. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index e5c8e24a9dbbfbc..16be593763bf667 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -3014,7 +3014,7 @@ EM_JS(int, emscripten_set_up_async_input_device_js, (void), { await sleep(5); return bufs[(idx ++) % 3]; }); - return !!WebAssembly.promising; + return !!Module._Py_EmscriptenStackSwitching; }); static PyObject * diff --git a/Platforms/emscripten/web_embed_test/main.c b/Platforms/emscripten/web_embed_test/main.c new file mode 100644 index 000000000000000..462024d86cb75df --- /dev/null +++ b/Platforms/emscripten/web_embed_test/main.c @@ -0,0 +1,50 @@ +/* Smoke test for libpython linked into a program whose main() is not Python's. */ + +#include <Python.h> +#include <stdio.h> + +// Imports come from the preloaded zip, len() goes through the call trampoline +// and poll() through the syscall overrides. +static const char *SCRIPT = + "import json, select\n" + "select.poll().poll(0)\n" + "print(json.dumps({'embedded': len('ok')}))\n"; + +int main(void) +{ + PyStatus status; + PyConfig config; + + PyConfig_InitIsolatedConfig(&config); + config.write_bytecode = 0; + config.module_search_paths_set = 1; + status = PyWideStringList_Append(&config.module_search_paths, + L"/lib/stdlib.zip"); + if (PyStatus_Exception(status)) { + goto exception; + } + status = PyConfig_SetBytesString(&config, &config.executable, "/embed"); + if (PyStatus_Exception(status)) { + goto exception; + } + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto exception; + } + PyConfig_Clear(&config); + + if (PyRun_SimpleString(SCRIPT) != 0) { + puts("web_embed_test: script failed"); + return 1; + } + if (Py_FinalizeEx() < 0) { + puts("web_embed_test: Py_FinalizeEx failed"); + return 1; + } + puts("web_embed_test: ok"); + return 0; + +exception: + PyConfig_Clear(&config); + Py_ExitStatusException(status); +} diff --git a/Platforms/emscripten/web_embed_test/run_test.sh b/Platforms/emscripten/web_embed_test/run_test.sh new file mode 100755 index 000000000000000..ecbbe73efd8b4f5 --- /dev/null +++ b/Platforms/emscripten/web_embed_test/run_test.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Build and run the libpython embedding smoke test. +set -euo pipefail +cd "$(dirname "$0")/../../.." + +BUILD_DIR=${CROSS_BUILD_DIR:-cross-build}/wasm32-emscripten/build/python +make -C "$BUILD_DIR" web_embed_test + +# Node 24 needs JSPI enabled explicitly; it is on by default afterwards. +NODE=${NODE:-node} +NODE_FLAGS= +if [ "$("$NODE" -p 'process.versions.node.split(".")[0]')" = 24 ]; then + NODE_FLAGS=--experimental-wasm-jspi +fi + +# Run from the build directory: Emscripten resolves main.data relative to cwd. +cd "$BUILD_DIR/web_embed_test" +rc=0 +out=$("$NODE" $NODE_FLAGS main.js 2>&1) || rc=$? +echo "$out" +[ "$rc" -eq 0 ] && grep -q "web_embed_test: ok" <<<"$out" diff --git a/Programs/emscripten_beforemain.c b/Programs/emscripten_beforemain.c new file mode 100644 index 000000000000000..445470237ea8b80 --- /dev/null +++ b/Programs/emscripten_beforemain.c @@ -0,0 +1,14 @@ +/* Emscripten setup for when Python is the program. Linked into the + * interpreter only, never into libpython. + * + * Runs main() under WebAssembly.promising so that libpython can suspend the + * wasm stack in a syscall (see Python/emscripten_syscalls.c). + */ + +void _PyEmscripten_BeforeMain_js(void); + +__attribute__((constructor)) void +_PyEmscripten_BeforeMain(void) +{ + _PyEmscripten_BeforeMain_js(); +} diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c index cdd5fc6e91e89bc..f0c677fa883a231 100644 --- a/Python/emscripten_syscalls.c +++ b/Python/emscripten_syscalls.c @@ -54,13 +54,40 @@ __attribute__((constructor)) void __syscall_init_umask(void) { #define EM_JS_MACROS(ret, func_name, args, body...) \ EM_JS(ret, func_name, args, body) -EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { - // Define FS.createAsyncInputDevice(), This is quite similar to - // FS.createDevice() defined here: - // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libfs.js?plain=1#L1642 - // but instead of returning one byte at a time, the input() function should - // return a Uint8Array. This makes the handler code simpler, the - // `createAsyncInputDevice` simpler, and everything faster. +EM_JS(void, _PyEmscripten_BeforeMain_js, (void), { + if (!WebAssembly.promising) { + // No stack switching support =( + return; + } + if (ENVIRONMENT_IS_NODE && !Module.onExit) { + Module.onExit = (code) => process.exit(code); + } + // promising() needs the raw export; _main may be a JS wrapper around it. + const main = WebAssembly.promising(wasmExports.__main_argc_argv); + _main = (...args) => { + Module._Py_EmscriptenStackSwitching = true; + (async () => { + const ret = await main(...args); + Module.onExit?.(ret); + })(); + _emscripten_exit_with_live_runtime(); + }; + // callMain() takes the entry point from _main, or from wasmImports.main + // when linked with -sMAIN_MODULE. + if ("main" in wasmImports) { + wasmImports.main = _main; + } +}) + +EM_JS_DEPS(_PyEmscripten_BeforeMain, "emscripten_exit_with_live_runtime"); + +// Define FS.createAsyncInputDevice(), This is quite similar to +// FS.createDevice() defined here: +// https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libfs.js?plain=1#L1642 +// but instead of returning one byte at a time, the input() function should +// return a Uint8Array. This makes the handler code simpler, the +// `createAsyncInputDevice` simpler, and everything faster. +EM_JS_MACROS(void, __fs_init_async_input_device_js, (void), { FS.createAsyncInputDevice = function(parent, name, input) { parent = typeof parent == 'string' ? parent : FS.getPath(parent); var path = PATH.join2(parent, name); @@ -103,44 +130,14 @@ EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { FS.registerDevice(dev, ops); return FS.mkdev(path, mode, dev); }; - if (!WebAssembly.promising) { - // No stack switching support =( - return; - } - const origResolveGlobalSymbol = resolveGlobalSymbol; - if (ENVIRONMENT_IS_NODE && !Module.onExit) { - Module.onExit = (code) => process.exit(code); - } - // * wrap the main symbol with WebAssembly.promising, - // * call exit_with_live_runtime() to prevent emscripten from shutting down - // the runtime before the promise resolves, - // * call onExit / process.exit ourselves, since exit_with_live_runtime() - // prevented Emscripten from calling it normally. - resolveGlobalSymbol = function (name, direct = false) { - const orig = origResolveGlobalSymbol(name, direct); - if (name === "main") { - const main = WebAssembly.promising(orig.sym); - orig.sym = (...args) => { - (async () => { - const ret = await main(...args); - Module.onExit?.(ret); - })(); - _emscripten_exit_with_live_runtime(); - }; - } - return orig; - }; }) -EM_JS_DEPS(_emscripten_promising_main, - "$FS,$PATH,$FS_getMode,$resolveGlobalSymbol," - "emscripten_exit_with_live_runtime"); +EM_JS_DEPS(__fs_init_async_input_device, "$FS,$PATH,$FS_getMode"); -__attribute__((constructor)) void _emscripten_promising_main(void) { - _emscripten_promising_main_js(); +__attribute__((constructor)) void __fs_init_async_input_device(void) { + __fs_init_async_input_device_js(); } - #define IOVEC_T_BUF_OFFSET 0 #define IOVEC_T_BUF_LEN_OFFSET 4 #define IOVEC_T_SIZE 8 @@ -154,13 +151,20 @@ _Static_assert(sizeof(__wasi_iovec_t) == IOVEC_T_SIZE, // If the stream has a readAsync handler, read to buffer defined in iovs, write // number of bytes read to *nread, and return a promise that resolves to the // errno. Otherwise, return null. +// +// Reading from an async input device and poll() suspend the wasm stack +// instead of blocking when main() runs under WebAssembly.promising, which +// Programs/emscripten_beforemain.c arranges for the interpreter. An embedder +// running its own promising entry point opts in with +// Module._Py_EmscriptenStackSwitching = true; +// Otherwise these calls keep their synchronous behavior. EM_JS_MACROS(__externref_t, __maybe_fd_read_async, ( __wasi_fd_t fd, const __wasi_iovec_t *iovs, size_t iovcnt, __wasi_size_t *nread ), { - if (!WebAssembly.promising) { + if (!Module._Py_EmscriptenStackSwitching) { return null; } var stream; @@ -244,7 +248,7 @@ _Static_assert(offsetof(struct pollfd, revents) == 6, "Unepxected pollfd struct _Static_assert(sizeof(struct pollfd) == 8, "Unepxected pollfd struct layout"); EM_JS_MACROS(__externref_t, __maybe_poll_async, (intptr_t fds, int nfds, int timeout), { - if (!WebAssembly.promising) { + if (!Module._Py_EmscriptenStackSwitching) { return null; } return (async function() { diff --git a/configure b/configure index 04260d2c4b58344..81af3fa0402a20f 100755 --- a/configure +++ b/configure @@ -878,6 +878,7 @@ TRUE MACHDEP_OBJS DYNLOADFILE DLINCLDIR +PLATFORM_MAIN_OBJS PLATFORM_OBJS PLATFORM_HEADERS DTRACE_OBJS @@ -20434,6 +20435,7 @@ fi PLATFORM_HEADERS= PLATFORM_OBJS= +PLATFORM_MAIN_OBJS= case $ac_sys_system in #( Emscripten) : @@ -20444,6 +20446,7 @@ case $ac_sys_system in #( if test "x$enable_emscripten_syscalls" = xyes then : + as_fn_append PLATFORM_MAIN_OBJS ' Programs/emscripten_beforemain.o' as_fn_append PLATFORM_OBJS ' Python/emscripten_syscalls.o' fi @@ -20460,6 +20463,7 @@ esac + # -I${DLINCLDIR} is added to the compile rule for importdl.o DLINCLDIR=. diff --git a/configure.ac b/configure.ac index b53247194fc9003..a55276ed5e6e8f8 100644 --- a/configure.ac +++ b/configure.ac @@ -5488,6 +5488,7 @@ fi dnl Platform-specific C and header files. PLATFORM_HEADERS= PLATFORM_OBJS= +PLATFORM_MAIN_OBJS= AS_CASE([$ac_sys_system], [Emscripten], [ @@ -5495,6 +5496,7 @@ AS_CASE([$ac_sys_system], AS_VAR_APPEND([CFLAGS_NODIST], [" -mgc"]) AS_VAR_IF([enable_emscripten_syscalls], [yes], [ + AS_VAR_APPEND([PLATFORM_MAIN_OBJS], [' Programs/emscripten_beforemain.o']) AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_syscalls.o']) ]) AS_VAR_APPEND([PLATFORM_HEADERS], [' $(srcdir)/Include/internal/pycore_emscripten_signal.h']) @@ -5506,6 +5508,7 @@ AS_CASE([$ac_sys_system], ) AC_SUBST([PLATFORM_HEADERS]) AC_SUBST([PLATFORM_OBJS]) +AC_SUBST([PLATFORM_MAIN_OBJS]) # -I${DLINCLDIR} is added to the compile rule for importdl.o AC_SUBST([DLINCLDIR]) _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
