Re: [vim/vim] Impossible to pass non-UTF-8 strings from vim to Python 3 (#1053)

Nikolay Aleksandrovich Pavlov Sun, 11 Sep 2016 09:09:26 -0700

I would suggest something like below, but I have no tests.

commit 615351832d75df3dfbc3f22694e675583e0b325d
Author: ZyX <[email protected]>
Date:   Tue Aug 16 21:42:24 2016 +0300


    Use surrogateescape when appropriate

diff --git a/src/if_py_both.h b/src/if_py_both.h
index 35ad5d0..6709300 100644
--- a/src/if_py_both.h
+++ b/src/if_py_both.h
@@ -134,7 +134,8 @@ StringToChars(PyObject *obj, PyObject **todecref)
     {
  PyObject *bytes;

- if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL)))
+ if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT,
+ ERRORS_ENCODE_ARG)))
     return NULL;

  if(PyBytes_AsStringAndSize(bytes, (char **) &str, NULL) == -1
@@ -4117,7 +4118,8 @@ StringToLine(PyObject *obj)
     }
     else if (PyUnicode_Check(obj))
     {
- if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL)))
+ if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT,
+ ERRORS_ENCODE_ARG)))
     return NULL;

  if (PyBytes_AsStringAndSize(bytes, &str, &len) == -1
@@ -6197,7 +6199,7 @@ _ConvertFromPyObject(PyObject *obj, typval_T
*tv, PyObject *lookup_dict)
  PyObject *bytes;
  char_u *str;

- bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL);
+ bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, ERRORS_ENCODE_ARG);
  if (bytes == NULL)
     return -1;

diff --git a/src/if_python.c b/src/if_python.c
index 622634d..1cafe34 100644
--- a/src/if_python.c
+++ b/src/if_python.c
@@ -70,6 +70,9 @@
 # undef PY_SSIZE_T_CLEAN
 #endif

+#define ERRORS_DECODE_ARG NULL
+#define ERRORS_ENCODE_ARG ERRORS_DECODE_ARG
+
 #if defined(MACOS) && !defined(MACOS_X_UNIX)
 # include "macglue.h"
 # include <CodeFragments.h>
diff --git a/src/if_python3.c b/src/if_python3.c
index 10984cd..9085c3c 100644
--- a/src/if_python3.c
+++ b/src/if_python3.c
@@ -91,12 +91,15 @@
 /* Python 3 does not support CObjects, always use Capsules */
 #define PY_USE_CAPSULE

+#define ERRORS_DECODE_ARG CODEC_ERROR_HANDLER
+#define ERRORS_ENCODE_ARG ERRORS_DECODE_ARG
+
 #define PyInt Py_ssize_t
 #ifndef PyString_Check
 # define PyString_Check(obj) PyUnicode_Check(obj)
 #endif
 #define PyString_FromString(repr) \
-    PyUnicode_Decode(repr, STRLEN(repr), ENC_OPT, NULL)
+    PyUnicode_Decode(repr, STRLEN(repr), ENC_OPT, ERRORS_DECODE_ARG)
 #define PyString_FromFormat PyUnicode_FromFormat
 #ifndef PyInt_Check
 # define PyInt_Check(obj) PyLong_Check(obj)
@@ -969,8 +972,8 @@ DoPyCommand(const char *cmd, rangeinitializer
init_range, runner run, void *arg)
     /* PyRun_SimpleString expects a UTF-8 string. Wrong encoding may cause
      * SyntaxError (unicode error). */
     cmdstr = PyUnicode_Decode(cmd, strlen(cmd),
- (char *)ENC_OPT, CODEC_ERROR_HANDLER);
-    cmdbytes = PyUnicode_AsEncodedString(cmdstr, "utf-8", CODEC_ERROR_HANDLER);
+ (char *)ENC_OPT, ERRORS_DECODE_ARG);
+    cmdbytes = PyUnicode_AsEncodedString(cmdstr, "utf-8", ERRORS_ENCODE_ARG);
     Py_XDECREF(cmdstr);

     run(PyBytes_AsString(cmdbytes), arg, &pygilstate);
@@ -1642,7 +1645,7 @@ LineToString(const char *str)
     }
     *p = '\0';

-    result = PyUnicode_Decode(tmp, len, (char *)ENC_OPT, CODEC_ERROR_HANDLER);
+    result = PyUnicode_Decode(tmp, len, (char *)ENC_OPT, ERRORS_DECODE_ARG);

     vim_free(tmp);
     return result;

-- 
-- 
You received this message from the "vim_dev" maillist.
Do not top-post! Type your reply below the text you are replying to.
For more information, visit http://www.vim.org/maillist.php

--- 
You received this message because you are subscribed to the Google Groups 
"vim_dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to [email protected].
For more options, visit https://groups.google.com/d/optout.

commit 615351832d75df3dfbc3f22694e675583e0b325d
Author: ZyX <[email protected]>
Date:   Tue Aug 16 21:42:24 2016 +0300

    Use surrogateescape when appropriate

diff --git a/src/if_py_both.h b/src/if_py_both.h
index 35ad5d0..6709300 100644
--- a/src/if_py_both.h
+++ b/src/if_py_both.h
@@ -134,7 +134,8 @@ StringToChars(PyObject *obj, PyObject **todecref)
     {
        PyObject        *bytes;
 
-       if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL)))
+       if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT,
+                       ERRORS_ENCODE_ARG)))
            return NULL;
 
        if(PyBytes_AsStringAndSize(bytes, (char **) &str, NULL) == -1
@@ -4117,7 +4118,8 @@ StringToLine(PyObject *obj)
     }
     else if (PyUnicode_Check(obj))
     {
-       if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL)))
+       if (!(bytes = PyUnicode_AsEncodedString(obj, ENC_OPT,
+                       ERRORS_ENCODE_ARG)))
            return NULL;
 
        if (PyBytes_AsStringAndSize(bytes, &str, &len) == -1
@@ -6197,7 +6199,7 @@ _ConvertFromPyObject(PyObject *obj, typval_T *tv, 
PyObject *lookup_dict)
        PyObject        *bytes;
        char_u  *str;
 
-       bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, NULL);
+       bytes = PyUnicode_AsEncodedString(obj, ENC_OPT, ERRORS_ENCODE_ARG);
        if (bytes == NULL)
            return -1;
 
diff --git a/src/if_python.c b/src/if_python.c
index 622634d..1cafe34 100644
--- a/src/if_python.c
+++ b/src/if_python.c
@@ -70,6 +70,9 @@
 # undef PY_SSIZE_T_CLEAN
 #endif
 
+#define ERRORS_DECODE_ARG NULL
+#define ERRORS_ENCODE_ARG ERRORS_DECODE_ARG
+
 #if defined(MACOS) && !defined(MACOS_X_UNIX)
 # include "macglue.h"
 # include <CodeFragments.h>
diff --git a/src/if_python3.c b/src/if_python3.c
index 10984cd..9085c3c 100644
--- a/src/if_python3.c
+++ b/src/if_python3.c
@@ -91,12 +91,15 @@
 /* Python 3 does not support CObjects, always use Capsules */
 #define PY_USE_CAPSULE
 
+#define ERRORS_DECODE_ARG CODEC_ERROR_HANDLER
+#define ERRORS_ENCODE_ARG ERRORS_DECODE_ARG
+
 #define PyInt Py_ssize_t
 #ifndef PyString_Check
 # define PyString_Check(obj) PyUnicode_Check(obj)
 #endif
 #define PyString_FromString(repr) \
-    PyUnicode_Decode(repr, STRLEN(repr), ENC_OPT, NULL)
+    PyUnicode_Decode(repr, STRLEN(repr), ENC_OPT, ERRORS_DECODE_ARG)
 #define PyString_FromFormat PyUnicode_FromFormat
 #ifndef PyInt_Check
 # define PyInt_Check(obj) PyLong_Check(obj)
@@ -969,8 +972,8 @@ DoPyCommand(const char *cmd, rangeinitializer init_range, 
runner run, void *arg)
     /* PyRun_SimpleString expects a UTF-8 string. Wrong encoding may cause
      * SyntaxError (unicode error). */
     cmdstr = PyUnicode_Decode(cmd, strlen(cmd),
-                                       (char *)ENC_OPT, CODEC_ERROR_HANDLER);
-    cmdbytes = PyUnicode_AsEncodedString(cmdstr, "utf-8", CODEC_ERROR_HANDLER);
+                                       (char *)ENC_OPT, ERRORS_DECODE_ARG);
+    cmdbytes = PyUnicode_AsEncodedString(cmdstr, "utf-8", ERRORS_ENCODE_ARG);
     Py_XDECREF(cmdstr);
 
     run(PyBytes_AsString(cmdbytes), arg, &pygilstate);
@@ -1642,7 +1645,7 @@ LineToString(const char *str)
     }
     *p = '\0';
 
-    result = PyUnicode_Decode(tmp, len, (char *)ENC_OPT, CODEC_ERROR_HANDLER);
+    result = PyUnicode_Decode(tmp, len, (char *)ENC_OPT, ERRORS_DECODE_ARG);
 
     vim_free(tmp);
     return result;

Re: [vim/vim] Impossible to pass non-UTF-8 strings from vim to Python 3 (#1053)

Raspunde prin e-mail lui