Author: Amaury Forgeot d'Arc <[email protected]>
Branch: py3k
Changeset: r59058:26d73ca4f4e4
Date: 2012-11-22 22:46 +0100
http://bitbucket.org/pypy/pypy/changeset/26d73ca4f4e4/
Log: cpyext: Implement the various FileSystem encoding functions.
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -121,6 +121,7 @@
METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O
Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS
Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE
+Py_CLEANUP_SUPPORTED
""".split()
for name in constant_names:
setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -103,13 +103,6 @@
raise NotImplementedError
-@cpython_api([PyObject], PyObject)
-def PyBytes_FromObject(space, o):
- """Return the bytes representation of object o that implements the buffer
- protocol."""
- raise NotImplementedError
-
-
@cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
def PyCell_Check(space, ob):
"""Return true if ob is a cell object; ob must not be NULL."""
@@ -2114,65 +2107,6 @@
raise NotImplementedError
-@cpython_api([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
-def PyUnicode_FSConverter(space, obj, result):
- """ParseTuple converter: encode str objects to bytes using
- PyUnicode_EncodeFSDefault(); bytes objects are output as-is.
- result must be a PyBytesObject* which must be released when it is
- no longer used.
- """
- raise NotImplementedError
-
-
-@cpython_api([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
-def PyUnicode_FSDecoder(space, obj, result):
- """ParseTuple converter: decode bytes objects to str using
- PyUnicode_DecodeFSDefaultAndSize(); str objects are output
- as-is. result must be a PyUnicodeObject* which must be released
- when it is no longer used.
- """
- raise NotImplementedError
-
-
-@cpython_api([rffi.CCHARP, Py_ssize_t], PyObject)
-def PyUnicode_DecodeFSDefaultAndSize(space, s, size):
- """Decode a string using Py_FileSystemDefaultEncoding and the
- 'surrogateescape' error handler, or 'strict' on Windows.
-
- If Py_FileSystemDefaultEncoding is not set, fall back to the
- locale encoding.
-
- Use 'strict' error handler on Windows."""
- raise NotImplementedError
-
-
-@cpython_api([rffi.CCHARP], PyObject)
-def PyUnicode_DecodeFSDefault(space, s):
- """Decode a null-terminated string using Py_FileSystemDefaultEncoding
- and the 'surrogateescape' error handler, or 'strict' on Windows.
-
- If Py_FileSystemDefaultEncoding is not set, fall back to the
- locale encoding.
-
- Use PyUnicode_DecodeFSDefaultAndSize() if you know the string length.
-
- Use 'strict' error handler on Windows."""
- raise NotImplementedError
-
-
-@cpython_api([PyObject], PyObject)
-def PyUnicode_EncodeFSDefault(space, unicode):
- """Encode a Unicode object to Py_FileSystemDefaultEncoding with the
- 'surrogateescape' error handler, or 'strict' on Windows, and return
- bytes. Note that the resulting bytes object may contain
- null bytes.
-
- If Py_FileSystemDefaultEncoding is not set, fall back to the
- locale encoding.
- """
- raise NotImplementedError
-
-
@cpython_api([rffi.CArrayPtr(Py_UNICODE), Py_ssize_t, rffi.CCHARP,
rffi.CCHARP], PyObject)
def PyUnicode_Encode(space, s, size, encoding, errors):
"""Encode the Py_UNICODE buffer s of the given size and return a Python
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py
b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -3,7 +3,7 @@
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
from pypy.module.cpyext.unicodeobject import (
Py_UNICODE, PyUnicodeObject, new_empty_unicode)
-from pypy.module.cpyext.api import PyObjectP, PyObject
+from pypy.module.cpyext.api import PyObjectP, PyObject, Py_CLEANUP_SUPPORTED
from pypy.module.cpyext.pyobject import Py_DecRef, from_ref
from pypy.rpython.lltypesystem import rffi, lltype
import sys, py
@@ -272,6 +272,42 @@
assert res == 0
assert s == "12ሴ"
+ def test_encode_fsdefault(self, space, api):
+ w_u = space.wrap(u'sp�m')
+ w_s = api.PyUnicode_EncodeFSDefault(w_u)
+ with rffi.scoped_str2charp(space.str_w(w_s)) as encoded:
+ w_decoded = api.PyUnicode_DecodeFSDefaultAndSize(encoded,
space.len_w(w_s))
+ assert space.eq_w(w_decoded, w_u)
+ w_decoded = api.PyUnicode_DecodeFSDefault(encoded)
+ assert space.eq_w(w_decoded, w_u)
+
+ def test_fsconverter(self, space, api):
+ # Input is bytes
+ w_input = space.wrapbytes("test")
+ with lltype.scoped_alloc(PyObjectP.TO, 1) as result:
+ # Decoder
+ ret = api.PyUnicode_FSDecoder(w_input, result)
+ assert ret == Py_CLEANUP_SUPPORTED
+ assert space.isinstance_w(from_ref(space, result[0]),
space.w_unicode)
+ assert api.PyUnicode_FSDecoder(None, result) == 1
+ # Converter
+ ret = api.PyUnicode_FSConverter(w_input, result)
+ assert ret == Py_CLEANUP_SUPPORTED
+ assert space.eq_w(from_ref(space, result[0]), w_input)
+ assert api.PyUnicode_FSDecoder(None, result) == 1
+ # Input is unicode
+ w_input = space.wrap("test")
+ with lltype.scoped_alloc(PyObjectP.TO, 1) as result:
+ # Decoder
+ ret = api.PyUnicode_FSDecoder(w_input, result)
+ assert ret == Py_CLEANUP_SUPPORTED
+ assert space.eq_w(from_ref(space, result[0]), w_input)
+ assert api.PyUnicode_FSDecoder(None, result) == 1
+ # Converter
+ ret = api.PyUnicode_FSConverter(w_input, result)
+ assert ret == Py_CLEANUP_SUPPORTED
+ assert space.isinstance_w(from_ref(space, result[0]),
space.w_bytes)
+ assert api.PyUnicode_FSDecoder(None, result) == 1
def test_IS(self, space, api):
for char in [0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1c, 0x1d, 0x1e, 0x1f,
diff --git a/pypy/module/cpyext/unicodeobject.py
b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -5,13 +5,15 @@
from pypy.module.cpyext.api import (
CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
bootstrap_function, PyObjectFields, cpython_struct, CONST_STRING,
- CONST_WSTRING)
+ CONST_WSTRING, Py_CLEANUP_SUPPORTED)
from pypy.module.cpyext.pyerrors import PyErr_BadArgument
from pypy.module.cpyext.pyobject import (
PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
make_typedescr, get_typedescr)
from pypy.module.cpyext.stringobject import PyString_Check
+from pypy.module.cpyext.bytesobject import PyBytes_FromObject
from pypy.module._codecs.interp_codecs import CodecState
+from pypy.module.posix.interp_posix import fsencode, fsdecode
from pypy.objspace.std import unicodeobject, unicodetype, stringtype
from pypy.rlib import runicode
from pypy.tool.sourcetools import func_renamer
@@ -406,6 +408,96 @@
space.wrap("decoding Unicode is not supported"))
return space.call_function(w_meth, w_encoding, w_errors)
+
+@cpython_api([PyObject, PyObjectP], rffi.INT_real, error=0)
+def PyUnicode_FSConverter(space, w_obj, result):
+ """ParseTuple converter: encode str objects to bytes using
+ PyUnicode_EncodeFSDefault(); bytes objects are output as-is.
+ result must be a PyBytesObject* which must be released when it is
+ no longer used.
+ """
+ if not w_obj:
+ # Implement ParseTuple cleanup support
+ Py_DecRef(space, result[0])
+ return 1
+ if space.isinstance_w(w_obj, space.w_bytes):
+ w_output = w_obj
+ else:
+ w_obj = PyUnicode_FromObject(space, w_obj)
+ w_output = fsencode(space, w_obj)
+ if not space.isinstance_w(w_output, space.w_bytes):
+ raise OperationError(space.w_TypeError,
+ space.wrap("encoder failed to return bytes"))
+ data = space.bytes0_w(w_output) # Check for NUL bytes
+ result[0] = make_ref(space, w_output)
+ return Py_CLEANUP_SUPPORTED
+
+
+@cpython_api([PyObject, PyObjectP], rffi.INT_real, error=0)
+def PyUnicode_FSDecoder(space, w_obj, result):
+ """ParseTuple converter: decode bytes objects to str using
+ PyUnicode_DecodeFSDefaultAndSize(); str objects are output
+ as-is. result must be a PyUnicodeObject* which must be released
+ when it is no longer used.
+ """
+ if not w_obj:
+ # Implement ParseTuple cleanup support
+ Py_DecRef(space, result[0])
+ return 1
+ if space.isinstance_w(w_obj, space.w_unicode):
+ w_output = w_obj
+ else:
+ w_obj = PyBytes_FromObject(space, w_obj)
+ w_output = fsdecode(space, w_obj)
+ if not space.isinstance_w(w_output, space.w_unicode):
+ raise OperationError(space.w_TypeError,
+ space.wrap("decoder failed to return
unicode"))
+ data = space.unicode0_w(w_output) # Check for NUL bytes
+ result[0] = make_ref(space, w_output)
+ return Py_CLEANUP_SUPPORTED
+
+
+@cpython_api([rffi.CCHARP, Py_ssize_t], PyObject)
+def PyUnicode_DecodeFSDefaultAndSize(space, s, size):
+ """Decode a string using Py_FileSystemDefaultEncoding and the
+ 'surrogateescape' error handler, or 'strict' on Windows.
+
+ If Py_FileSystemDefaultEncoding is not set, fall back to the
+ locale encoding.
+
+ Use 'strict' error handler on Windows."""
+ w_bytes = space.wrapbytes(rffi.charpsize2str(s, size))
+ return fsdecode(space, w_bytes)
+
+
+@cpython_api([rffi.CCHARP], PyObject)
+def PyUnicode_DecodeFSDefault(space, s):
+ """Decode a null-terminated string using Py_FileSystemDefaultEncoding
+ and the 'surrogateescape' error handler, or 'strict' on Windows.
+
+ If Py_FileSystemDefaultEncoding is not set, fall back to the
+ locale encoding.
+
+ Use PyUnicode_DecodeFSDefaultAndSize() if you know the string length.
+
+ Use 'strict' error handler on Windows."""
+ w_bytes = space.wrapbytes(rffi.charp2str(s))
+ return fsdecode(space, w_bytes)
+
+
+@cpython_api([PyObject], PyObject)
+def PyUnicode_EncodeFSDefault(space, w_unicode):
+ """Encode a Unicode object to Py_FileSystemDefaultEncoding with the
+ 'surrogateescape' error handler, or 'strict' on Windows, and return
+ bytes. Note that the resulting bytes object may contain
+ null bytes.
+
+ If Py_FileSystemDefaultEncoding is not set, fall back to the
+ locale encoding.
+ """
+ return fsencode(space, w_unicode)
+
+
@cpython_api([CONST_STRING], PyObject)
def PyUnicode_FromString(space, s):
"""Create a Unicode object from an UTF-8 encoded null-terminated char
buffer"""
diff --git a/pypy/module/posix/interp_posix.py
b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -38,11 +38,15 @@
def fsencode_w(space, w_obj):
if space.isinstance_w(w_obj, space.w_unicode):
- w_obj = space.call_method(w_obj, 'encode',
- getfilesystemencoding(space),
- space.wrap('surrogateescape'))
+ w_obj = fsencode(space, w_obj)
return space.bytes0_w(w_obj)
+def fsencode(space, w_obj):
+ w_bytes = space.call_method(w_obj, 'encode',
+ getfilesystemencoding(space),
+ space.wrap('surrogateescape'))
+ return w_bytes
+
def fsdecode(space, w_obj):
w_unicode = space.call_method(w_obj, 'decode',
getfilesystemencoding(space),
_______________________________________________
pypy-commit mailing list
[email protected]
http://mail.python.org/mailman/listinfo/pypy-commit
