On Mon, Jun 13, 2011 at 03:54:04AM +0200, Bram Moolenaar wrote:
> I have included the patch, but on my Ubuntu system I get this error:
>
> E448: Could not load library function PyUnicode_UCSX_*
>
> when doing something simple as ":py3 print('hello')"
>
> Can you fix that?
>
Yes, just a typo preventing Python symbols from loading.
Later a user found that there was heavy memory leakage, and I tried hard
to fix them. It was partly introduced by my code, and I believe partly
had been there before. Attached patch should fix this too.
--
Best regards,
lilydjwg
--
You received this message from the "vim_dev" maillist.
Do not top-post! Type your reply below the text you are replying to.
For more information, visit http://www.vim.org/maillist.php
diff --git a/src/if_py_both.h b/src/if_py_both.h
index f78eccb..72ec8b0 100644
--- a/src/if_py_both.h
+++ b/src/if_py_both.h
@@ -65,10 +65,10 @@ static struct PyMethodDef OutputMethods[] = {
OutputWrite(PyObject *self, PyObject *args)
{
int len;
- char *str;
+ char *str = NULL;
int error = ((OutputObject *)(self))->error;
- if (!PyArg_ParseTuple(args, "s#", &str, &len))
+ if (!PyArg_ParseTuple(args, "es#", p_enc, &str, &len))
return NULL;
Py_BEGIN_ALLOW_THREADS
@@ -76,6 +76,7 @@ OutputWrite(PyObject *self, PyObject *args)
writer((writefn)(error ? emsg : msg), (char_u *)str, len);
Python_Release_Vim();
Py_END_ALLOW_THREADS
+ PyMem_Free(str);
Py_INCREF(Py_None);
return Py_None;
@@ -104,10 +105,10 @@ OutputWritelines(PyObject *self, PyObject *args)
for (i = 0; i < n; ++i)
{
PyObject *line = PyList_GetItem(list, i);
- char *str;
+ char *str = NULL;
PyInt len;
- if (!PyArg_Parse(line, "s#", &str, &len)) {
+ if (!PyArg_Parse(line, "es#", p_enc, &str, &len)) {
PyErr_SetString(PyExc_TypeError, _("writelines() requires list of
strings"));
Py_DECREF(list);
return NULL;
@@ -118,6 +119,7 @@ OutputWritelines(PyObject *self, PyObject *args)
writer((writefn)(error ? emsg : msg), (char_u *)str, len);
Python_Release_Vim();
Py_END_ALLOW_THREADS
+ PyMem_Free(str);
}
Py_DECREF(list);
@@ -681,6 +683,7 @@ StringToLine(PyObject *obj)
{
const char *str;
char *save;
+ PyObject *bytes;
PyInt len;
PyInt i;
char *p;
@@ -691,8 +694,9 @@ StringToLine(PyObject *obj)
return NULL;
}
- str = PyString_AsString(obj);
- len = PyString_Size(obj);
+ bytes = PyString_AsBytes(obj);
+ str = PyString_AsString(bytes);
+ len = PyString_Size(bytes);
/*
* Error checking: String must not contain newlines, as we
@@ -731,6 +735,7 @@ StringToLine(PyObject *obj)
}
save[i] = '\0';
+ PyString_FreeBytes(bytes);
return save;
}
@@ -908,6 +913,193 @@ SetBufferLine(buf_T *buf, PyInt n, PyObject *line, PyInt
*len_change)
}
}
+/* Replace a range of lines in the specified buffer. The line numbers are in
+ * Vim format (1-based). The range is from lo up to, but not including, hi.
+ * The replacement lines are given as a Python list of string objects. The
+ * list is checked for validity and correct format. Errors are returned as a
+ * value of FAIL. The return value is OK on success.
+ * If OK is returned and len_change is not NULL, *len_change
+ * is set to the change in the buffer length.
+ */
+ static int
+SetBufferLineList(buf_T *buf, PyInt lo, PyInt hi, PyObject *list, PyInt
*len_change)
+{
+ /* First of all, we check the thpe of the supplied Python object.
+ * There are three cases:
+ * 1. NULL, or None - this is a deletion.
+ * 2. A list - this is a replacement.
+ * 3. Anything else - this is an error.
+ */
+ if (list == Py_None || list == NULL)
+ {
+ PyInt i;
+ PyInt n = (int)(hi - lo);
+ buf_T *savebuf = curbuf;
+
+ PyErr_Clear();
+ curbuf = buf;
+
+ if (u_savedel((linenr_T)lo, (long)n) == FAIL)
+ PyErr_SetVim(_("cannot save undo information"));
+ else
+ {
+ for (i = 0; i < n; ++i)
+ {
+ if (ml_delete((linenr_T)lo, FALSE) == FAIL)
+ {
+ PyErr_SetVim(_("cannot delete line"));
+ break;
+ }
+ }
+ if (buf == curwin->w_buffer)
+ py_fix_cursor((linenr_T)lo, (linenr_T)hi, (linenr_T)-n);
+ deleted_lines_mark((linenr_T)lo, (long)i);
+ }
+
+ curbuf = savebuf;
+
+ if (PyErr_Occurred() || VimErrorCheck())
+ return FAIL;
+
+ if (len_change)
+ *len_change = -n;
+
+ return OK;
+ }
+ else if (PyList_Check(list))
+ {
+ PyInt i;
+ PyInt new_len = PyList_Size(list);
+ PyInt old_len = hi - lo;
+ PyInt extra = 0; /* lines added to text, can be negative */
+ char **array;
+ buf_T *savebuf;
+
+ if (new_len == 0) /* avoid allocating zero bytes */
+ array = NULL;
+ else
+ {
+ array = (char **)alloc((unsigned)(new_len * sizeof(char *)));
+ if (array == NULL)
+ {
+ PyErr_NoMemory();
+ return FAIL;
+ }
+ }
+
+ for (i = 0; i < new_len; ++i)
+ {
+ PyObject *line = PyList_GetItem(list, i);
+
+ array[i] = StringToLine(line);
+ if (array[i] == NULL)
+ {
+ while (i)
+ vim_free(array[--i]);
+ vim_free(array);
+ return FAIL;
+ }
+ }
+
+ savebuf = curbuf;
+
+ PyErr_Clear();
+ curbuf = buf;
+
+ if (u_save((linenr_T)(lo-1), (linenr_T)hi) == FAIL)
+ PyErr_SetVim(_("cannot save undo information"));
+
+ /* If the size of the range is reducing (ie, new_len < old_len) we
+ * need to delete some old_len. We do this at the start, by
+ * repeatedly deleting line "lo".
+ */
+ if (!PyErr_Occurred())
+ {
+ for (i = 0; i < old_len - new_len; ++i)
+ if (ml_delete((linenr_T)lo, FALSE) == FAIL)
+ {
+ PyErr_SetVim(_("cannot delete line"));
+ break;
+ }
+ extra -= i;
+ }
+
+ /* For as long as possible, replace the existing old_len with the
+ * new old_len. This is a more efficient operation, as it requires
+ * less memory allocation and freeing.
+ */
+ if (!PyErr_Occurred())
+ {
+ for (i = 0; i < old_len && i < new_len; ++i)
+ if (ml_replace((linenr_T)(lo+i), (char_u *)array[i], FALSE)
+ == FAIL)
+ {
+ PyErr_SetVim(_("cannot replace line"));
+ break;
+ }
+ }
+ else
+ i = 0;
+
+ /* Now we may need to insert the remaining new old_len. If we do, we
+ * must free the strings as we finish with them (we can't pass the
+ * responsibility to vim in this case).
+ */
+ if (!PyErr_Occurred())
+ {
+ while (i < new_len)
+ {
+ if (ml_append((linenr_T)(lo + i - 1),
+ (char_u *)array[i], 0, FALSE) == FAIL)
+ {
+ PyErr_SetVim(_("cannot insert line"));
+ break;
+ }
+ vim_free(array[i]);
+ ++i;
+ ++extra;
+ }
+ }
+
+ /* Free any left-over old_len, as a result of an error */
+ while (i < new_len)
+ {
+ vim_free(array[i]);
+ ++i;
+ }
+
+ /* Free the array of old_len. All of its contents have now
+ * been dealt with (either freed, or the responsibility passed
+ * to vim.
+ */
+ vim_free(array);
+
+ /* Adjust marks. Invalidate any which lie in the
+ * changed range, and move any in the remainder of the buffer.
+ */
+ mark_adjust((linenr_T)lo, (linenr_T)(hi - 1),
+ (long)MAXLNUM, (long)extra);
+ changed_lines((linenr_T)lo, 0, (linenr_T)hi, (long)extra);
+
+ if (buf == curwin->w_buffer)
+ py_fix_cursor((linenr_T)lo, (linenr_T)hi, (linenr_T)extra);
+
+ curbuf = savebuf;
+
+ if (PyErr_Occurred() || VimErrorCheck())
+ return FAIL;
+
+ if (len_change)
+ *len_change = new_len - old_len;
+
+ return OK;
+ }
+ else
+ {
+ PyErr_BadArgument();
+ return FAIL;
+ }
+}
/* Insert a number of lines into the specified buffer after the specifed line.
* The line number is in Vim format (1-based). The lines to be inserted are
@@ -1113,6 +1305,40 @@ RBAsItem(BufferObject *self, PyInt n, PyObject *val,
PyInt start, PyInt end, PyI
return 0;
}
+ static PyInt
+RBAsSlice(BufferObject *self, PyInt lo, PyInt hi, PyObject *val, PyInt start,
PyInt end, PyInt *new_end)
+{
+ PyInt size;
+ PyInt len_change;
+
+ /* Self must be a valid buffer */
+ if (CheckBuffer(self))
+ return -1;
+
+ /* Sort out the slice range */
+ size = end - start + 1;
+
+ if (lo < 0)
+ lo = 0;
+ else if (lo > size)
+ lo = size;
+ if (hi < 0)
+ hi = 0;
+ if (hi < lo)
+ hi = lo;
+ else if (hi > size)
+ hi = size;
+
+ if (SetBufferLineList(self->buf, lo + start, hi + start,
+ val, &len_change) == FAIL)
+ return -1;
+
+ if (new_end)
+ *new_end = end + len_change;
+
+ return 0;
+}
+
static PyObject *
RBAppend(BufferObject *self, PyObject *args, PyInt start, PyInt end, PyInt
*new_end)
diff --git a/src/if_python.c b/src/if_python.c
index d35e9ab..dd9d6da 100644
--- a/src/if_python.c
+++ b/src/if_python.c
@@ -56,6 +56,9 @@
static void init_structs(void);
+#define PyString_AsBytes(obj) (obj)
+#define PyString_FreeBytes(obj) (bytes)
+
#if !defined(FEAT_PYTHON) && defined(PROTO)
/* Use this to be able to generate prototypes without python being used. */
# define PyObject Py_ssize_t
@@ -129,6 +132,7 @@ struct PyMethodDef { Py_ssize_t a; };
*/
# define PyArg_Parse dll_PyArg_Parse
# define PyArg_ParseTuple dll_PyArg_ParseTuple
+# define PyMem_Free dll_PyMem_Free
# define PyDict_SetItemString dll_PyDict_SetItemString
# define PyErr_BadArgument dll_PyErr_BadArgument
# define PyErr_Clear dll_PyErr_Clear
@@ -189,6 +193,7 @@ struct PyMethodDef { Py_ssize_t a; };
*/
static int(*dll_PyArg_Parse)(PyObject *, char *, ...);
static int(*dll_PyArg_ParseTuple)(PyObject *, char *, ...);
+static int(*dll_PyMem_Free)(void *);
static int(*dll_PyDict_SetItemString)(PyObject *dp, char *key, PyObject *item);
static int(*dll_PyErr_BadArgument)(void);
static void(*dll_PyErr_Clear)(void);
@@ -271,6 +276,7 @@ static struct
{
{"PyArg_Parse", (PYTHON_PROC*)&dll_PyArg_Parse},
{"PyArg_ParseTuple", (PYTHON_PROC*)&dll_PyArg_ParseTuple},
+ {"PyMem_Free", (PYTHON_PROC*)&dll_PyMem_Free},
{"PyDict_SetItemString", (PYTHON_PROC*)&dll_PyDict_SetItemString},
{"PyErr_BadArgument", (PYTHON_PROC*)&dll_PyErr_BadArgument},
{"PyErr_Clear", (PYTHON_PROC*)&dll_PyErr_Clear},
@@ -833,44 +839,6 @@ static PyInt RangeAssSlice(PyObject *, PyInt, PyInt,
PyObject *);
static PyObject *CurrentGetattr(PyObject *, char *);
static int CurrentSetattr(PyObject *, char *, PyObject *);
-/* Common routines for buffers and line ranges
- * -------------------------------------------
- */
-
- static PyInt
-RBAssSlice(BufferObject *self, PyInt lo, PyInt hi, PyObject *val, PyInt start,
PyInt end, PyInt *new_end)
-{
- PyInt size;
- PyInt len_change;
-
- /* Self must be a valid buffer */
- if (CheckBuffer(self))
- return -1;
-
- /* Sort out the slice range */
- size = end - start + 1;
-
- if (lo < 0)
- lo = 0;
- else if (lo > size)
- lo = size;
- if (hi < 0)
- hi = 0;
- if (hi < lo)
- hi = lo;
- else if (hi > size)
- hi = size;
-
- if (SetBufferLineList(self->buf, lo + start, hi + start,
- val, &len_change) == FAIL)
- return -1;
-
- if (new_end)
- *new_end = end + len_change;
-
- return 0;
-}
-
static PySequenceMethods BufferAsSeq = {
(PyInquiry) BufferLength, /* sq_length, len(x) */
(binaryfunc) 0, /* BufferConcat, */ /* sq_concat, x+y
*/
@@ -1038,7 +1006,7 @@ BufferAssItem(PyObject *self, PyInt n, PyObject *val)
static PyInt
BufferAssSlice(PyObject *self, PyInt lo, PyInt hi, PyObject *val)
{
- return RBAssSlice((BufferObject *)(self), lo, hi, val, 1,
+ return RBAsSlice((BufferObject *)(self), lo, hi, val, 1,
(PyInt)((BufferObject *)(self))->buf->b_ml.ml_line_count,
NULL);
}
@@ -1088,7 +1056,7 @@ RangeAssItem(PyObject *self, PyInt n, PyObject *val)
static PyInt
RangeAssSlice(PyObject *self, PyInt lo, PyInt hi, PyObject *val)
{
- return RBAssSlice(((RangeObject *)(self))->buf, lo, hi, val,
+ return RBAsSlice(((RangeObject *)(self))->buf, lo, hi, val,
((RangeObject *)(self))->start,
((RangeObject *)(self))->end,
&((RangeObject *)(self))->end);
@@ -1435,194 +1403,6 @@ PythonMod_Init(void)
* 4. Utility functions for handling the interface between Vim and Python.
*/
-/* Replace a range of lines in the specified buffer. The line numbers are in
- * Vim format (1-based). The range is from lo up to, but not including, hi.
- * The replacement lines are given as a Python list of string objects. The
- * list is checked for validity and correct format. Errors are returned as a
- * value of FAIL. The return value is OK on success.
- * If OK is returned and len_change is not NULL, *len_change
- * is set to the change in the buffer length.
- */
- static int
-SetBufferLineList(buf_T *buf, PyInt lo, PyInt hi, PyObject *list, PyInt
*len_change)
-{
- /* First of all, we check the thpe of the supplied Python object.
- * There are three cases:
- * 1. NULL, or None - this is a deletion.
- * 2. A list - this is a replacement.
- * 3. Anything else - this is an error.
- */
- if (list == Py_None || list == NULL)
- {
- PyInt i;
- PyInt n = (int)(hi - lo);
- buf_T *savebuf = curbuf;
-
- PyErr_Clear();
- curbuf = buf;
-
- if (u_savedel((linenr_T)lo, (long)n) == FAIL)
- PyErr_SetVim(_("cannot save undo information"));
- else
- {
- for (i = 0; i < n; ++i)
- {
- if (ml_delete((linenr_T)lo, FALSE) == FAIL)
- {
- PyErr_SetVim(_("cannot delete line"));
- break;
- }
- }
- if (buf == curwin->w_buffer)
- py_fix_cursor((linenr_T)lo, (linenr_T)hi, (linenr_T)-n);
- deleted_lines_mark((linenr_T)lo, (long)i);
- }
-
- curbuf = savebuf;
-
- if (PyErr_Occurred() || VimErrorCheck())
- return FAIL;
-
- if (len_change)
- *len_change = -n;
-
- return OK;
- }
- else if (PyList_Check(list))
- {
- PyInt i;
- PyInt new_len = PyList_Size(list);
- PyInt old_len = hi - lo;
- PyInt extra = 0; /* lines added to text, can be negative */
- char **array;
- buf_T *savebuf;
-
- if (new_len == 0) /* avoid allocating zero bytes */
- array = NULL;
- else
- {
- array = (char **)alloc((unsigned)(new_len * sizeof(char *)));
- if (array == NULL)
- {
- PyErr_NoMemory();
- return FAIL;
- }
- }
-
- for (i = 0; i < new_len; ++i)
- {
- PyObject *line = PyList_GetItem(list, i);
-
- array[i] = StringToLine(line);
- if (array[i] == NULL)
- {
- while (i)
- vim_free(array[--i]);
- vim_free(array);
- return FAIL;
- }
- }
-
- savebuf = curbuf;
-
- PyErr_Clear();
- curbuf = buf;
-
- if (u_save((linenr_T)(lo-1), (linenr_T)hi) == FAIL)
- PyErr_SetVim(_("cannot save undo information"));
-
- /* If the size of the range is reducing (ie, new_len < old_len) we
- * need to delete some old_len. We do this at the start, by
- * repeatedly deleting line "lo".
- */
- if (!PyErr_Occurred())
- {
- for (i = 0; i < old_len - new_len; ++i)
- if (ml_delete((linenr_T)lo, FALSE) == FAIL)
- {
- PyErr_SetVim(_("cannot delete line"));
- break;
- }
- extra -= i;
- }
-
- /* For as long as possible, replace the existing old_len with the
- * new old_len. This is a more efficient operation, as it requires
- * less memory allocation and freeing.
- */
- if (!PyErr_Occurred())
- {
- for (i = 0; i < old_len && i < new_len; ++i)
- if (ml_replace((linenr_T)(lo+i), (char_u *)array[i], FALSE)
- == FAIL)
- {
- PyErr_SetVim(_("cannot replace line"));
- break;
- }
- }
- else
- i = 0;
-
- /* Now we may need to insert the remaining new old_len. If we do, we
- * must free the strings as we finish with them (we can't pass the
- * responsibility to vim in this case).
- */
- if (!PyErr_Occurred())
- {
- while (i < new_len)
- {
- if (ml_append((linenr_T)(lo + i - 1),
- (char_u *)array[i], 0, FALSE) == FAIL)
- {
- PyErr_SetVim(_("cannot insert line"));
- break;
- }
- vim_free(array[i]);
- ++i;
- ++extra;
- }
- }
-
- /* Free any left-over old_len, as a result of an error */
- while (i < new_len)
- {
- vim_free(array[i]);
- ++i;
- }
-
- /* Free the array of old_len. All of its contents have now
- * been dealt with (either freed, or the responsibility passed
- * to vim.
- */
- vim_free(array);
-
- /* Adjust marks. Invalidate any which lie in the
- * changed range, and move any in the remainder of the buffer.
- */
- mark_adjust((linenr_T)lo, (linenr_T)(hi - 1),
- (long)MAXLNUM, (long)extra);
- changed_lines((linenr_T)lo, 0, (linenr_T)hi, (long)extra);
-
- if (buf == curwin->w_buffer)
- py_fix_cursor((linenr_T)lo, (linenr_T)hi, (linenr_T)extra);
-
- curbuf = savebuf;
-
- if (PyErr_Occurred() || VimErrorCheck())
- return FAIL;
-
- if (len_change)
- *len_change = new_len - old_len;
-
- return OK;
- }
- else
- {
- PyErr_BadArgument();
- return FAIL;
- }
-}
-
/* Convert a Vim line into a Python string.
* All internal newlines are replaced by null characters.
*
diff --git a/src/if_python3.c b/src/if_python3.c
index b253796..800ec83 100644
--- a/src/if_python3.c
+++ b/src/if_python3.c
@@ -70,8 +70,10 @@ static void init_structs(void);
#define PyInt Py_ssize_t
#define PyString_Check(obj) PyUnicode_Check(obj)
-#define PyString_AsString(obj) _PyUnicode_AsString(obj)
-#define PyString_Size(obj) PyUnicode_GET_SIZE(obj)
+#define PyString_AsBytes(obj) PyUnicode_AsEncodedString(obj, p_enc, NULL);
+#define PyString_FreeBytes(obj) Py_XDECREF(bytes)
+#define PyString_AsString(obj) PyBytes_AsString(obj)
+#define PyString_Size(obj) PyBytes_GET_SIZE(bytes)
#define PyString_FromString(repr) PyUnicode_FromString(repr)
#if defined(DYNAMIC_PYTHON3) || defined(PROTO)
@@ -99,6 +101,7 @@ static void init_structs(void);
# define PyArg_Parse py3_PyArg_Parse
# undef PyArg_ParseTuple
# define PyArg_ParseTuple py3_PyArg_ParseTuple
+# define PyMem_Free py3_PyMem_Free
# define PyDict_SetItemString py3_PyDict_SetItemString
# define PyErr_BadArgument py3_PyErr_BadArgument
# define PyErr_Clear py3_PyErr_Clear
@@ -140,8 +143,13 @@ static void init_structs(void);
# define PyModule_AddObject py3_PyModule_AddObject
# define PyImport_AppendInittab py3_PyImport_AppendInittab
# define _PyUnicode_AsString py3__PyUnicode_AsString
+# undef PyUnicode_AsEncodedString
+# define PyUnicode_AsEncodedString py3_PyUnicode_AsEncodedString
+# undef PyBytes_AsString
+# define PyBytes_AsString py3_PyBytes_AsString
# define PyObject_GenericGetAttr py3_PyObject_GenericGetAttr
# define PySlice_Type (*py3_PySlice_Type)
+# define PyErr_NewException py3_PyErr_NewException
# ifdef Py_DEBUG
# define _Py_NegativeRefcount py3__Py_NegativeRefcount
# define _Py_RefTotal (*py3__Py_RefTotal)
@@ -157,8 +165,8 @@ static void init_structs(void);
# define PyModule_Create2 py3_PyModule_Create2
# undef PyUnicode_FromString
# define PyUnicode_FromString py3_PyUnicode_FromString
-# undef PyUnicode_FromStringAndSize
-# define PyUnicode_FromStringAndSize py3_PyUnicode_FromStringAndSize
+# undef PyUnicode_Decode
+# define PyUnicode_Decode py3_PyUnicode_Decode
# ifdef Py_DEBUG
# undef PyObject_NEW
@@ -199,7 +207,8 @@ static PyObject* (*py3_Py_BuildValue)(char *, ...);
static int (*py3_PyType_Ready)(PyTypeObject *type);
static int (*py3_PyDict_SetItemString)(PyObject *dp, char *key, PyObject
*item);
static PyObject* (*py3_PyUnicode_FromString)(const char *u);
-static PyObject* (*py3_PyUnicode_FromStringAndSize)(const char *u, Py_ssize_t
size);
+static PyObject* (*py3_PyUnicode_Decode)(const char *u, Py_ssize_t size,
+ const char *encoding, const char *errors);
static long (*py3_PyLong_AsLong)(PyObject *);
static void (*py3_PyErr_SetNone)(PyObject *);
static void (*py3_PyEval_InitThreads)(void);
@@ -207,6 +216,7 @@ static void(*py3_PyEval_RestoreThread)(PyThreadState *);
static PyThreadState*(*py3_PyEval_SaveThread)(void);
static int (*py3_PyArg_Parse)(PyObject *, char *, ...);
static int (*py3_PyArg_ParseTuple)(PyObject *, char *, ...);
+static int (*py3_PyMem_Free)(void *);
static int (*py3_Py_IsInitialized)(void);
static void (*py3_PyErr_Clear)(void);
static PyObject*(*py3__PyObject_Init)(PyObject *, PyTypeObject *);
@@ -214,11 +224,14 @@ static PyObject* py3__Py_NoneStruct;
static int (*py3_PyModule_AddObject)(PyObject *m, const char *name, PyObject
*o);
static int (*py3_PyImport_AppendInittab)(const char *name, PyObject*
(*initfunc)(void));
static char* (*py3__PyUnicode_AsString)(PyObject *unicode);
+static PyObject* (*py3_PyUnicode_AsEncodedString)(PyObject *unicode, const
char* encoding, const char* errors);
+static char* (*py3_PyBytes_AsString)(PyObject *bytes);
static PyObject* (*py3_PyObject_GenericGetAttr)(PyObject *obj, PyObject *name);
static PyObject* (*py3_PyModule_Create2)(struct PyModuleDef* module, int
module_api_version);
static PyObject* (*py3_PyType_GenericAlloc)(PyTypeObject *type, Py_ssize_t
nitems);
static PyObject* (*py3_PyType_GenericNew)(PyTypeObject *type, PyObject *args,
PyObject *kwds);
static PyTypeObject* py3_PySlice_Type;
+static PyObject* (*py3_PyErr_NewException)(char *name, PyObject *base,
PyObject *dict);
# ifdef Py_DEBUG
static void (*py3__Py_NegativeRefcount)(const char *fname, int lineno,
PyObject *op);
static Py_ssize_t* py3__Py_RefTotal;
@@ -259,6 +272,7 @@ static struct
{"Py_SetPythonHome", (PYTHON_PROC*)&py3_Py_SetPythonHome},
{"Py_Initialize", (PYTHON_PROC*)&py3_Py_Initialize},
{"PyArg_ParseTuple", (PYTHON_PROC*)&py3_PyArg_ParseTuple},
+ {"PyMem_Free", (PYTHON_PROC*)&py3_PyMem_Free},
{"PyList_New", (PYTHON_PROC*)&py3_PyList_New},
{"PyGILState_Ensure", (PYTHON_PROC*)&py3_PyGILState_Ensure},
{"PyGILState_Release", (PYTHON_PROC*)&py3_PyGILState_Release},
@@ -289,7 +303,6 @@ static struct
{"PyEval_RestoreThread", (PYTHON_PROC*)&py3_PyEval_RestoreThread},
{"PyEval_SaveThread", (PYTHON_PROC*)&py3_PyEval_SaveThread},
{"PyArg_Parse", (PYTHON_PROC*)&py3_PyArg_Parse},
- {"PyArg_ParseTuple", (PYTHON_PROC*)&py3_PyArg_ParseTuple},
{"Py_IsInitialized", (PYTHON_PROC*)&py3_Py_IsInitialized},
{"_Py_NoneStruct", (PYTHON_PROC*)&py3__Py_NoneStruct},
{"PyErr_Clear", (PYTHON_PROC*)&py3_PyErr_Clear},
@@ -297,11 +310,13 @@ static struct
{"PyModule_AddObject", (PYTHON_PROC*)&py3_PyModule_AddObject},
{"PyImport_AppendInittab", (PYTHON_PROC*)&py3_PyImport_AppendInittab},
{"_PyUnicode_AsString", (PYTHON_PROC*)&py3__PyUnicode_AsString},
+ {"PyBytes_AsString", (PYTHON_PROC*)&py3_PyBytes_AsString},
{"PyObject_GenericGetAttr", (PYTHON_PROC*)&py3_PyObject_GenericGetAttr},
{"PyModule_Create2", (PYTHON_PROC*)&py3_PyModule_Create2},
{"PyType_GenericAlloc", (PYTHON_PROC*)&py3_PyType_GenericAlloc},
{"PyType_GenericNew", (PYTHON_PROC*)&py3_PyType_GenericNew},
{"PySlice_Type", (PYTHON_PROC*)&py3_PySlice_Type},
+ {"PyErr_NewException", (PYTHON_PROC*)&py3_PyErr_NewException},
# ifdef Py_DEBUG
{"_Py_NegativeRefcount", (PYTHON_PROC*)&py3__Py_NegativeRefcount},
{"_Py_RefTotal", (PYTHON_PROC*)&py3__Py_RefTotal},
@@ -337,7 +352,7 @@ end_dynamic_python3(void)
py3_runtime_link_init(char *libname, int verbose)
{
int i;
- void *ucs_from_string, *ucs_from_string_and_size;
+ void *ucs_from_string, *ucs_decode, *ucs_as_encoded_string;
# if !(defined(PY_NO_RTLD_GLOBAL) && defined(PY3_NO_RTLD_GLOBAL)) &&
defined(UNIX) && defined(FEAT_PYTHON)
/* Can't have Python and Python3 loaded at the same time.
@@ -377,19 +392,24 @@ py3_runtime_link_init(char *libname, int verbose)
/* Load unicode functions separately as only the ucs2 or the ucs4 functions
* will be present in the library. */
ucs_from_string = symbol_from_dll(hinstPy3, "PyUnicodeUCS2_FromString");
- ucs_from_string_and_size = symbol_from_dll(hinstPy3,
- "PyUnicodeUCS2_FromStringAndSize");
- if (!ucs_from_string || !ucs_from_string_and_size)
+ ucs_decode = symbol_from_dll(hinstPy3,
+ "PyUnicodeUCS2_Decode");
+ ucs_as_encoded_string = symbol_from_dll(hinstPy3,
+ "PyUnicodeUCS2_AsEncodedString");
+ if (!ucs_from_string || !ucs_decode || !ucs_as_encoded_string)
{
ucs_from_string = symbol_from_dll(hinstPy3,
"PyUnicodeUCS4_FromString");
- ucs_from_string_and_size = symbol_from_dll(hinstPy3,
- "PyUnicodeUCS4_FromStringAndSize");
+ ucs_decode = symbol_from_dll(hinstPy3,
+ "PyUnicodeUCS4_Decode");
+ ucs_as_encoded_string = symbol_from_dll(hinstPy3,
+ "PyUnicodeUCS4_AsEncodedString");
}
- if (ucs_from_string && ucs_from_string_and_size)
+ if (ucs_from_string && ucs_decode && ucs_as_encoded_string)
{
py3_PyUnicode_FromString = ucs_from_string;
- py3_PyUnicode_FromStringAndSize = ucs_from_string_and_size;
+ py3_PyUnicode_Decode = ucs_decode;
+ py3_PyUnicode_AsEncodedString = ucs_as_encoded_string;
}
else
{
@@ -568,8 +591,11 @@ Python3_Init(void)
/* Remove the element from sys.path that was added because of our
* argv[0] value in Py3Init_vim(). Previously we used an empty
* string, but dependinding on the OS we then get an empty entry or
- * the current directory in sys.path. */
- PyRun_SimpleString("import sys; sys.path = list(filter(lambda x: x !=
'/must>not&exist', sys.path))");
+ * the current directory in sys.path.
+ * Only after vim has been imported, the element does exist in
+ * sys.path.
+ */
+ PyRun_SimpleString("import vim; import sys; sys.path =
list(filter(lambda x: not x.endswith('must>not&exist'), sys.path))");
// lock is created and acquired in PyEval_InitThreads() and thread
// state is created in Py_Initialize()
@@ -606,6 +632,8 @@ DoPy3Command(exarg_T *eap, const char *cmd)
#if defined(HAVE_LOCALE_H) || defined(X_LOCALE)
char *saved_locale;
#endif
+ PyObject *cmdstr;
+ PyObject *cmdbytes;
#if defined(MACOS) && !defined(MACOS_X_UNIX)
GetPort(&oldPort);
@@ -635,7 +663,13 @@ DoPy3Command(exarg_T *eap, const char *cmd)
pygilstate = PyGILState_Ensure();
- PyRun_SimpleString((char *)(cmd));
+ /* PyRun_SimpleString expects a UTF-8 string. Wrong encoding may cause
+ * SyntaxError (unicode error). */
+ cmdstr = PyUnicode_Decode(cmd, strlen(cmd), p_enc, NULL);
+ cmdbytes = PyUnicode_AsEncodedString(cmdstr, "utf-8", NULL);
+ Py_XDECREF(cmdstr);
+ PyRun_SimpleString(PyBytes_AsString(cmdbytes));
+ Py_XDECREF(cmdbytes);
PyGILState_Release(pygilstate);
@@ -694,7 +728,10 @@ ex_py3file(exarg_T *eap)
* different options under Windows, meaning that stdio pointers aren't
* compatible between the two. Yuk.
*
- * construct: exec(compile(open('a_filename').read(), 'a_filename',
'exec'))
+ * construct: exec(compile(open('a_filename', 'rb').read(), 'a_filename',
'exec'))
+ *
+ * Using bytes so that Python can detect the source encoding as it normally
+ * does. The doc does not say "compile" accept bytes, though.
*
* We need to escape any backslashes or single quotes in the file name, so
that
* Python won't mangle the file name.
@@ -717,8 +754,8 @@ ex_py3file(exarg_T *eap)
return;
if (i==0)
{
- strcpy(p,"').read(),'");
- p += 11;
+ strcpy(p,"','rb').read(),'");
+ p += 16;
}
else
{
@@ -813,8 +850,8 @@ PythonIO_Fini(void)
static Py_ssize_t BufferLength(PyObject *);
static PyObject *BufferItem(PyObject *, Py_ssize_t);
-static Py_ssize_t BufferAsItem(PyObject *, Py_ssize_t, PyObject *);
static PyObject* BufferSubscript(PyObject *self, PyObject* idx);
+static Py_ssize_t BufferAsSubscript(PyObject *self, PyObject* idx, PyObject*
val);
/* Line range type - Implementation functions
@@ -836,7 +873,7 @@ static PySequenceMethods BufferAsSeq = {
(ssizeargfunc) 0, /* sq_repeat, x*n */
(ssizeargfunc) BufferItem, /* sq_item, x[i] */
0, /* was_sq_slice, x[i:j] */
- (ssizeobjargproc) BufferAsItem, /* sq_ass_item, x[i]=v */
+ 0, /* sq_ass_item, x[i]=v */
0, /* sq_ass_slice, x[i:j]=v */
0, /* sq_contains */
0, /* sq_inplace_concat */
@@ -846,7 +883,7 @@ static PySequenceMethods BufferAsSeq = {
PyMappingMethods BufferAsMapping = {
/* mp_length */ (lenfunc)BufferLength,
/* mp_subscript */ (binaryfunc)BufferSubscript,
- /* mp_ass_subscript */ (objobjargproc)0,
+ /* mp_ass_subscript */ (objobjargproc)BufferAsSubscript,
};
@@ -898,6 +935,8 @@ BufferDestructor(PyObject *self)
if (this->buf && this->buf != INVALID_BUFFER_VALUE)
this->buf->b_python3_ref = NULL;
+
+ Py_TYPE(self)->tp_free((PyObject*)self);
}
static PyObject *
@@ -976,15 +1015,6 @@ BufferSlice(PyObject *self, Py_ssize_t lo, Py_ssize_t hi)
(Py_ssize_t)((BufferObject *)(self))->buf->b_ml.ml_line_count);
}
- static Py_ssize_t
-BufferAsItem(PyObject *self, Py_ssize_t n, PyObject *val)
-{
- return RBAsItem((BufferObject *)(self), n, val, 1,
- (Py_ssize_t)((BufferObject *)(self))->buf->b_ml.ml_line_count,
- NULL);
-}
-
-
static PyObject *
BufferSubscript(PyObject *self, PyObject* idx)
{
@@ -994,19 +1024,45 @@ BufferSubscript(PyObject *self, PyObject* idx)
} else if (PySlice_Check(idx)) {
Py_ssize_t start, stop, step, slicelen;
- if (PySlice_GetIndicesEx((PySliceObject *)idx,
+ if (PySlice_GetIndicesEx(idx,
(Py_ssize_t)((BufferObject *)(self))->buf->b_ml.ml_line_count+1,
&start, &stop,
&step, &slicelen) < 0) {
return NULL;
}
- return BufferSlice(self,start,stop+1);
+ return BufferSlice(self,start,stop);
} else {
PyErr_SetString(PyExc_IndexError, "Index must be int or slice");
return NULL;
}
}
+ static Py_ssize_t
+BufferAsSubscript(PyObject *self, PyObject* idx, PyObject* val)
+{
+ if (PyLong_Check(idx)) {
+ long n = PyLong_AsLong(idx);
+ return RBAsItem((BufferObject *)(self), n, val, 1,
+ (Py_ssize_t)((BufferObject
*)(self))->buf->b_ml.ml_line_count,
+ NULL);
+ } else if (PySlice_Check(idx)) {
+ Py_ssize_t start, stop, step, slicelen;
+
+ if (PySlice_GetIndicesEx(idx,
+ (Py_ssize_t)((BufferObject *)(self))->buf->b_ml.ml_line_count+1,
+ &start, &stop,
+ &step, &slicelen) < 0) {
+ return -1;
+ }
+ return RBAsSlice((BufferObject *)(self), start, stop, val, 1,
+ (PyInt)((BufferObject
*)(self))->buf->b_ml.ml_line_count,
+ NULL);
+ } else {
+ PyErr_SetString(PyExc_IndexError, "Index must be int or slice");
+ return -1;
+ }
+}
+
static PySequenceMethods RangeAsSeq = {
(lenfunc) RangeLength, /* sq_length, len(x) */
(binaryfunc) 0, /* RangeConcat, sq_concat, x+y */
@@ -1033,6 +1089,7 @@ PyMappingMethods RangeAsMapping = {
RangeDestructor(PyObject *self)
{
Py_DECREF(((RangeObject *)(self))->buf);
+ Py_TYPE(self)->tp_free((PyObject*)self);
}
static PyObject *
@@ -1070,7 +1127,7 @@ RangeSubscript(PyObject *self, PyObject* idx)
} else if (PySlice_Check(idx)) {
Py_ssize_t start, stop, step, slicelen;
- if (PySlice_GetIndicesEx((PySliceObject *)idx,
+ if (PySlice_GetIndicesEx(idx,
((RangeObject *)(self))->end-((RangeObject *)(self))->start+1,
&start, &stop,
&step, &slicelen) < 0) {
@@ -1160,6 +1217,8 @@ WindowDestructor(PyObject *self)
if (this->win && this->win != INVALID_WINDOW_VALUE)
this->win->w_python3_ref = NULL;
+
+ Py_TYPE(self)->tp_free((PyObject*)self);
}
static PyObject *
@@ -1351,8 +1410,11 @@ PyMODINIT_FUNC Py3Init_vim(void)
PySys_SetArgv(1, argv);
mod = PyModule_Create(&vimmodule);
+ if (mod == NULL)
+ return NULL;
- VimError = Py_BuildValue("s", "vim.error");
+ VimError = PyErr_NewException("vim.error", NULL, NULL);
+ Py_INCREF(VimError);
PyModule_AddObject(mod, "error", VimError);
Py_INCREF((PyObject *)(void *)&TheBufferList);
@@ -1405,7 +1467,7 @@ LineToString(const char *str)
}
*p = '\0';
- result = PyUnicode_FromStringAndSize(tmp, len);
+ result = PyUnicode_Decode(tmp, len, p_enc, NULL);
vim_free(tmp);
return result;
