patch 9.0.1980: win32: issues with stable python ABI
Commit:
https://github.com/vim/vim/commit/119fdd9293f63614ed2ca60a78993466435db639
Author: Ken Takata <ken...@csc.jp>
Date: Wed Oct 4 20:05:05 2023 +0200
patch 9.0.1980: win32: issues with stable python ABI
Problem: win32: issues with stable python ABI
Solution: if_python3,win32: Fix Python3 stable ABI
There were some issues in current stable ABI implementation on Windows:
* Python DLL name should be `python3.dll` instead of `python311.dll` and
so on. (See: https://docs.python.org/3/c-api/stable.html)
* Some non-stable API functions were used:
- `_PyObject_NextNotImplemented`
- `PyStdPrinter_Type`
* `reset_stdin()` and `hook_py_exit()` didn't work with `python3.dll`.
`python3.dll` is a special type of DLL called forwarder DLL.
It just forwards the functions to other DLL (e.g. `python311.dll`).
There were two issues regarding these functions:
- `python3.dll` doesn't have import tables. This caused a crash in
`get_imported_func_info()`. Add a check whether the specified DLL
has an import table.
- `reset_stdin()` and `hook_py_exit()` should be applied to the
forwarded DLL (e.g. `python311.dll`), not to `python3.dll`.
Check the export directory of `python3.dll` to find the forwarded
DLL and apply the functions to it.
closes: #13260
Signed-off-by: Christian Brabandt <c...@256bit.org>
Co-authored-by: Ken Takata <ken...@csc.jp>
diff --git a/src/Make_cyg_ming.mak b/src/Make_cyg_ming.mak
index e4fe0be8f..98c86a6e7 100644
--- a/src/Make_cyg_ming.mak
+++ b/src/Make_cyg_ming.mak
@@ -394,16 +394,21 @@ DYNAMIC_PYTHON3=yes
ifndef PYTHON3_VER
PYTHON3_VER=36
+ endif
+ ifeq ($(DYNAMIC_PYTHON3_STABLE_ABI),yes)
+PYTHON3_NAME=python3
+ else
+PYTHON3_NAME=python$(PYTHON3_VER)
endif
ifndef DYNAMIC_PYTHON3_DLL
-DYNAMIC_PYTHON3_DLL=python$(PYTHON3_VER).dll
+DYNAMIC_PYTHON3_DLL=$(PYTHON3_NAME).dll
endif
ifdef PYTHON3_HOME
PYTHON3_HOME_DEF=-DPYTHON3_HOME=L\"$(PYTHON3_HOME)\"
endif
ifeq (no,$(DYNAMIC_PYTHON3))
-PYTHON3LIB=-L$(PYTHON3)/libs -lpython$(PYTHON3_VER)
+PYTHON3LIB=-L$(PYTHON3)/libs -l$(PYTHON3_NAME)
endif
ifndef PYTHON3INC
diff --git a/src/Make_mvc.mak b/src/Make_mvc.mak
index e1d42fdcb..618821d69 100644
--- a/src/Make_mvc.mak
+++ b/src/Make_mvc.mak
@@ -937,8 +937,13 @@ PYTHON_LIB = "$(PYTHON)\libs\python$(PYTHON_VER).lib"
! ifndef PYTHON3_VER
PYTHON3_VER = 36
! endif
+! if "$(DYNAMIC_PYTHON3_STABLE_ABI)" == "yes"
+PYTHON3_NAME = python3
+! else
+PYTHON3_NAME = python$(PYTHON3_VER)
+! endif
! ifndef DYNAMIC_PYTHON3_DLL
-DYNAMIC_PYTHON3_DLL = python$(PYTHON3_VER).dll
+DYNAMIC_PYTHON3_DLL = $(PYTHON3_NAME).dll
! endif
! message Python3 requested (version $(PYTHON3_VER)) - root dir is "$(PYTHON3)"
! if "$(DYNAMIC_PYTHON3)" == "yes"
@@ -953,13 +958,11 @@ CFLAGS = $(CFLAGS) -DDYNAMIC_PYTHON3 \
! if "$(DYNAMIC_PYTHON3_STABLE_ABI)" == "yes"
CFLAGS = $(CFLAGS) -DDYNAMIC_PYTHON3_STABLE_ABI
PYTHON3_INC = $(PYTHON3_INC) -DPy_LIMITED_API=0x3080000
-PYTHON3_LIB = /nodefaultlib:python3.lib
-! else
-PYTHON3_LIB = /nodefaultlib:python$(PYTHON3_VER).lib
! endif
+PYTHON3_LIB = /nodefaultlib:$(PYTHON3_NAME).lib
! else
CFLAGS = $(CFLAGS) -DPYTHON3_DLL=\"$(DYNAMIC_PYTHON3_DLL)\"
-PYTHON3_LIB = "$(PYTHON3)\libs\python$(PYTHON3_VER).lib"
+PYTHON3_LIB = "$(PYTHON3)\libs\$(PYTHON3_NAME).lib"
! endif
!endif
diff --git a/src/if_python3.c b/src/if_python3.c
index 2e116620e..bbbebc39f 100644
--- a/src/if_python3.c
+++ b/src/if_python3.c
@@ -242,7 +242,7 @@ static HINSTANCE hinstPy3 = 0; // Instance of python.dll
# if PY_VERSION_HEX >= 0x03040000
# define PyType_GetFlags py3_PyType_GetFlags
# endif
-#undef Py_BuildValue
+# undef Py_BuildValue
# define Py_BuildValue py3_Py_BuildValue
# define Py_SetPythonHome py3_Py_SetPythonHome
# define Py_Initialize py3_Py_Initialize
@@ -251,7 +251,9 @@ static HINSTANCE hinstPy3 = 0; // Instance of python.dll
# define _Py_NoneStruct (*py3__Py_NoneStruct)
# define _Py_FalseStruct (*py3__Py_FalseStruct)
# define _Py_TrueStruct (*py3__Py_TrueStruct)
-# define _PyObject_NextNotImplemented (*py3__PyObject_NextNotImplemented)
+# ifndef USE_LIMITED_API
+# define _PyObject_NextNotImplemented (*py3__PyObject_NextNotImplemented)
+# endif
# define PyModule_AddObject py3_PyModule_AddObject
# define PyImport_AppendInittab py3_PyImport_AppendInittab
# define PyImport_AddModule py3_PyImport_AddModule
@@ -288,7 +290,9 @@ static HINSTANCE hinstPy3 = 0; // Instance of python.dll
# define PyFloat_AsDouble py3_PyFloat_AsDouble
# define PyObject_GenericGetAttr py3_PyObject_GenericGetAttr
# define PyType_Type (*py3_PyType_Type)
-# define PyStdPrinter_Type (*py3_PyStdPrinter_Type)
+# ifndef USE_LIMITED_API
+# define PyStdPrinter_Type (*py3_PyStdPrinter_Type)
+# endif
# define PySlice_Type (*py3_PySlice_Type)
# define PyFloat_Type (*py3_PyFloat_Type)
# define PyNumber_Check (*py3_PyNumber_Check)
@@ -449,7 +453,9 @@ static void (*py3_PyErr_Clear)(void);
static PyObject* (*py3_PyErr_Format)(PyObject *, const char *, ...);
static void (*py3_PyErr_PrintEx)(int);
static PyObject*(*py3__PyObject_Init)(PyObject *, PyTypeObject *);
+# ifndef USE_LIMITED_API
static iternextfunc py3__PyObject_NextNotImplemented;
+# endif
static PyObject* py3__Py_NoneStruct;
static PyObject* py3__Py_FalseStruct;
static PyObject* py3__Py_TrueStruct;
@@ -485,7 +491,9 @@ static PyObject* (*py3_PyObject_GenericGetAttr)(PyObject
*obj, PyObject *name);
static PyObject* (*py3_PyType_GenericAlloc)(PyTypeObject *type, Py_ssize_t
nitems);
static PyObject* (*py3_PyType_GenericNew)(PyTypeObject *type, PyObject *args,
PyObject *kwds);
static PyTypeObject* py3_PyType_Type;
+# ifndef USE_LIMITED_API
static PyTypeObject* py3_PyStdPrinter_Type;
+# endif
static PyTypeObject* py3_PySlice_Type;
static PyTypeObject* py3_PyFloat_Type;
PyTypeObject* py3_PyBool_Type;
@@ -633,7 +641,9 @@ static struct
{"PyEval_SaveThread", (PYTHON_PROC*)&py3_PyEval_SaveThread},
{"_PyArg_Parse_SizeT", (PYTHON_PROC*)&py3_PyArg_Parse},
{"Py_IsInitialized", (PYTHON_PROC*)&py3_Py_IsInitialized},
+# ifndef USE_LIMITED_API
{"_PyObject_NextNotImplemented",
(PYTHON_PROC*)&py3__PyObject_NextNotImplemented},
+# endif
{"_Py_NoneStruct", (PYTHON_PROC*)&py3__Py_NoneStruct},
{"_Py_FalseStruct", (PYTHON_PROC*)&py3__Py_FalseStruct},
{"_Py_TrueStruct", (PYTHON_PROC*)&py3__Py_TrueStruct},
@@ -681,7 +691,9 @@ static struct
{"PyType_GenericAlloc", (PYTHON_PROC*)&py3_PyType_GenericAlloc},
{"PyType_GenericNew", (PYTHON_PROC*)&py3_PyType_GenericNew},
{"PyType_Type", (PYTHON_PROC*)&py3_PyType_Type},
+# ifndef USE_LIMITED_API
{"PyStdPrinter_Type", (PYTHON_PROC*)&py3_PyStdPrinter_Type},
+# endif
{"PySlice_Type", (PYTHON_PROC*)&py3_PySlice_Type},
{"PyFloat_Type", (PYTHON_PROC*)&py3_PyFloat_Type},
# if PY_VERSION_HEX < 0x030c00b0
@@ -1167,7 +1179,7 @@ reset_stdin(void)
{
FILE *(*py__acrt_iob_func)(unsigned) = NULL;
FILE *(*pyfreopen)(const char *, const char *, FILE *) = NULL;
- HINSTANCE hinst = hinstPy3;
+ HINSTANCE hinst = get_forwarded_dll(hinstPy3);
if (hinst == NULL || is_stdin_readable())
return;
@@ -1219,7 +1231,7 @@ hooked_exit(int ret)
static void
hook_py_exit(void)
{
- HINSTANCE hinst = hinstPy3;
+ HINSTANCE hinst = get_forwarded_dll(hinstPy3);
if (hinst == NULL || orig_exit != NULL)
return;
diff --git a/src/os_win32.c b/src/os_win32.c
index 7891944da..92bc8c157 100644
--- a/src/os_win32.c
+++ b/src/os_win32.c
@@ -626,15 +626,20 @@ get_imported_func_info(HINSTANCE hInst, const char
*funcname, int info,
PIMAGE_THUNK_DATA pIAT; // Import Address Table
PIMAGE_THUNK_DATA pINT; // Import Name Table
PIMAGE_IMPORT_BY_NAME pImpName;
+ DWORD ImpVA;
if (pDOS->e_magic != IMAGE_DOS_SIGNATURE)
return NULL;
pPE = (PIMAGE_NT_HEADERS)(pImage + pDOS->e_lfanew);
if (pPE->Signature != IMAGE_NT_SIGNATURE)
return NULL;
- pImpDesc = (PIMAGE_IMPORT_DESCRIPTOR)(pImage
- + pPE->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT]
- .VirtualAddress);
+
+ ImpVA = pPE->OptionalHeader
+ .DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT].VirtualAddress;
+ if (ImpVA == 0)
+ return NULL; // No Import Table
+ pImpDesc = (PIMAGE_IMPORT_DESCRIPTOR)(pImage + ImpVA);
+
for (; pImpDesc->FirstThunk; ++pImpDesc)
{
if (!pImpDesc->OriginalFirstThunk)
@@ -709,6 +714,65 @@ hook_dll_import_func(HINSTANCE hInst, const char
*funcname, const void *hook)
}
#endif
+#if defined(FEAT_PYTHON3) || defined(PROTO)
+/*
+ * Check if the specified DLL is a function forwarder.
+ * If yes, return the instance of the forwarded DLL.
+ * If no, return the specified DLL.
+ * If error, return NULL.
+ * This assumes that the DLL forwards all the function to a single DLL.
+ */
+ HINSTANCE
+get_forwarded_dll(HINSTANCE hInst)
+{
+ PBYTE pImage = (PBYTE)hInst;
+ PIMAGE_DOS_HEADER pDOS = (PIMAGE_DOS_HEADER)hInst;
+ PIMAGE_NT_HEADERS pPE;
+ PIMAGE_EXPORT_DIRECTORY pExpDir;
+ DWORD ExpVA;
+ DWORD ExpSize;
+ LPDWORD pFunctionTable;
+
+ if (pDOS->e_magic != IMAGE_DOS_SIGNATURE)
+ return NULL;
+ pPE = (PIMAGE_NT_HEADERS)(pImage + pDOS->e_lfanew);
+ if (pPE->Signature != IMAGE_NT_SIGNATURE)
+ return NULL;
+
+ ExpVA = pPE->OptionalHeader
+ .DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT].VirtualAddress;
+ ExpSize = pPE->OptionalHeader
+ .DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT].Size;
+ if (ExpVA == 0)
+ return hInst; // No Export Directory
+ pExpDir = (PIMAGE_EXPORT_DIRECTORY)(pImage + ExpVA);
+ pFunctionTable = (LPDWORD)(pImage + pExpDir->AddressOfFunctions);
+
+ if (pExpDir->NumberOfNames == 0)
+ return hInst; // No export names.
+
+ // Check only the first entry.
+ if ((pFunctionTable[0] < ExpVA) || (pFunctionTable[0] >= ExpVA + ExpSize))
+ // The first entry is not a function forwarder.
+ return hInst;
+
+ // The first entry is a function forwarder.
+ // The name is represented as "DllName.FunctionName".
+ const char *name = (const char *)(pImage + pFunctionTable[0]);
+ const char *p = strchr(name, '.');
+ if (p == NULL)
+ return hInst;
+
+ // Extract DllName.
+ char buf[MAX_PATH];
+ if (p - name + 1 > sizeof(buf))
+ return NULL;
+ strncpy(buf, name, p - name);
+ buf[p - name] = '
+ return GetModuleHandleA(buf);
+}
+#endif
+
#if defined(DYNAMIC_GETTEXT) || defined(PROTO)
# ifndef GETTEXT_DLL
# define GETTEXT_DLL "libintl.dll"
diff --git a/src/proto/os_win32.pro b/src/proto/os_win32.pro
index 7ea538848..cb8458531 100644
--- a/src/proto/os_win32.pro
+++ b/src/proto/os_win32.pro
@@ -5,6 +5,7 @@ int mch_is_gui_executable(void);
HINSTANCE find_imported_module_by_funcname(HINSTANCE hInst, const char
*funcname);
void *get_dll_import_func(HINSTANCE hInst, const char *funcname);
void *hook_dll_import_func(HINSTANCE hInst, const char *funcname, const void
*hook);
+HINSTANCE get_forwarded_dll(HINSTANCE hInst);
int dyn_libintl_init(void);
void dyn_libintl_end(void);
void PlatformId(void);
diff --git a/src/version.c b/src/version.c
index ba461ba11..d7b3c9bd3 100644
--- a/src/version.c
+++ b/src/version.c
@@ -704,6 +704,8 @@ static char *(features[]) =
static int included_patches[] =
{ /* Add new patch number below this line */
+/**/
+ 1980,
/**/
1979,
/**/
--
--
You received this message from the "vim_dev" maillist.
Do not top-post! Type your reply below the text you are replying to.
For more information, visit http://www.vim.org/maillist.php
---
You received this message because you are subscribed to the Google Groups
"vim_dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to vim_dev+unsubscr...@googlegroups.com.
To view this discussion on the web visit
https://groups.google.com/d/msgid/vim_dev/E1qo6Ol-007igk-MT%40256bit.org.
