Author: Ronan Lamy <ronan.l...@gmail.com> Branch: Changeset: r93648:f560bd567d72 Date: 2018-01-10 20:34 +0000 http://bitbucket.org/pypy/pypy/changeset/f560bd567d72/
Log: Backport rpython/ changes made on the py3.5 branch diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -381,16 +381,14 @@ class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), pairtype(SomeUnicodeCodePoint, SomeChar)): def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() + no_nul = uchr1.no_nul and uchr2.no_nul + return SomeUnicodeCodePoint(no_nul=no_nul) class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)): def union((uchr1, uchr2)): no_nul = uchr1.no_nul and uchr2.no_nul return SomeUnicodeCodePoint(no_nul=no_nul) - def add((chr1, chr2)): - return SomeUnicodeString() - class __extend__(pairtype(SomeString, SomeUnicodeString), pairtype(SomeUnicodeString, SomeString)): def mod((str, unistring)): diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -228,7 +228,9 @@ res = rwin32.LoadLibrary(name) if not res: err = rwin32.GetLastError_saved() - raise DLOpenError(rwin32.FormatError(err)) + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) return res def dlclose(handle): diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -2574,3 +2574,160 @@ """Passes offset==NULL; not support on all OSes""" res = c_sendfile(out_fd, in_fd, lltype.nullptr(_OFF_PTR_T.TO), count) return handle_posix_error('sendfile', res) + +# ____________________________________________________________ +# Support for *xattr functions + +if sys.platform.startswith('linux'): + + class CConfig: + _compilation_info_ = ExternalCompilationInfo( + includes=['sys/xattr.h', 'linux/limits.h'],) + XATTR_SIZE_MAX = rffi_platform.DefinedConstantInteger('XATTR_SIZE_MAX') + XATTR_CREATE = rffi_platform.DefinedConstantInteger('XATTR_CREATE') + XATTR_REPLACE = rffi_platform.DefinedConstantInteger('XATTR_REPLACE') + + cConfig = rffi_platform.configure(CConfig) + globals().update(cConfig) + c_fgetxattr = external('fgetxattr', + [rffi.INT, rffi.CCHARP, rffi.VOIDP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_getxattr = external('getxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.VOIDP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_lgetxattr = external('lgetxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.VOIDP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_fsetxattr = external('fsetxattr', + [rffi.INT, rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T, rffi.INT], + rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_setxattr = external('setxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T, rffi.INT], + rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_lsetxattr = external('lsetxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T, rffi.INT], + rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_fremovexattr = external('fremovexattr', + [rffi.INT, rffi.CCHARP], rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_removexattr = external('removexattr', + [rffi.CCHARP, rffi.CCHARP], rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_lremovexattr = external('lremovexattr', + [rffi.CCHARP, rffi.CCHARP], rffi.INT, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_flistxattr = external('flistxattr', + [rffi.INT, rffi.CCHARP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_listxattr = external('listxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + c_llistxattr = external('llistxattr', + [rffi.CCHARP, rffi.CCHARP, rffi.SIZE_T], rffi.SSIZE_T, + compilation_info=CConfig._compilation_info_, + save_err=rffi.RFFI_SAVE_ERRNO) + buf_sizes = [256, XATTR_SIZE_MAX] + + def fgetxattr(fd, name): + for size in buf_sizes: + with rffi.scoped_alloc_buffer(size) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + res = c_fgetxattr(fd, name, void_buf, size) + if res < 0: + err = get_saved_errno() + if err != errno.ERANGE: + raise OSError(err, 'fgetxattr failed') + else: + return buf.str(res) + else: + raise OSError(errno.ERANGE, 'fgetxattr failed') + + def getxattr(path, name, follow_symlinks=True): + for size in buf_sizes: + with rffi.scoped_alloc_buffer(size) as buf: + void_buf = rffi.cast(rffi.VOIDP, buf.raw) + if follow_symlinks: + res = c_getxattr(path, name, void_buf, size) + else: + res = c_lgetxattr(path, name, void_buf, size) + if res < 0: + err = get_saved_errno() + if err != errno.ERANGE: + c_name = 'getxattr' if follow_symlinks else 'lgetxattr' + raise OSError(err, c_name + 'failed') + else: + return buf.str(res) + else: + c_name = 'getxattr' if follow_symlinks else 'lgetxattr' + raise OSError(errno.ERANGE, c_name + 'failed') + + def fsetxattr(fd, name, value, flags=0): + return handle_posix_error( + 'fsetxattr', c_fsetxattr(fd, name, value, len(value), flags)) + + def setxattr(path, name, value, flags=0, follow_symlinks=True): + if follow_symlinks: + return handle_posix_error( + 'setxattr', c_setxattr(path, name, value, len(value), flags)) + else: + return handle_posix_error( + 'lsetxattr', c_lsetxattr(path, name, value, len(value), flags)) + + def fremovexattr(fd, name): + return handle_posix_error('fremovexattr', c_fremovexattr(fd, name)) + + def removexattr(path, name, follow_symlinks=True): + if follow_symlinks: + return handle_posix_error('removexattr', c_removexattr(path, name)) + else: + return handle_posix_error('lremovexattr', c_lremovexattr(path, name)) + + def _unpack_attrs(attr_string): + result = attr_string.split('\0') + del result[-1] + return result + + def flistxattr(fd): + for size in buf_sizes: + with rffi.scoped_alloc_buffer(size) as buf: + res = c_flistxattr(fd, buf.raw, size) + if res < 0: + err = get_saved_errno() + if err != errno.ERANGE: + raise OSError(err, 'flistxattr failed') + else: + return _unpack_attrs(buf.str(res)) + else: + raise OSError(errno.ERANGE, 'flistxattr failed') + + def listxattr(path, follow_symlinks=True): + for size in buf_sizes: + with rffi.scoped_alloc_buffer(size) as buf: + if follow_symlinks: + res = c_listxattr(path, buf.raw, size) + else: + res = c_llistxattr(path, buf.raw, size) + if res < 0: + err = get_saved_errno() + if err != errno.ERANGE: + c_name = 'listxattr' if follow_symlinks else 'llistxattr' + raise OSError(err, c_name + 'failed') + else: + return _unpack_attrs(buf.str(res)) + else: + c_name = 'listxattr' if follow_symlinks else 'llistxattr' + raise OSError(errno.ERANGE, c_name + 'failed') diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py --- a/rpython/rlib/rposix_scandir.py +++ b/rpython/rlib/rposix_scandir.py @@ -1,56 +1,126 @@ from rpython.rlib import rposix, rwin32 from rpython.rlib.objectmodel import specialize from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.rarithmetic import intmask -@specialize.argtype(0) -def opendir(path): - path = rposix._as_bytes0(path) - return opendir_bytes(path) - -def opendir_bytes(path): - dirp = rposix.c_opendir(path) - if not dirp: - raise OSError(rposix.get_saved_errno(), "opendir failed") - return dirp - -def closedir(dirp): - rposix.c_closedir(dirp) - if not rwin32.WIN32: + @specialize.argtype(0) + def opendir(path): + path = rposix._as_bytes0(path) + return opendir_bytes(path) + + def opendir_bytes(path): + dirp = rposix.c_opendir(path) + if not dirp: + raise OSError(rposix.get_saved_errno(), "opendir failed") + return dirp + + def closedir(dirp): + rposix.c_closedir(dirp) + NULL_DIRP = lltype.nullptr(rposix.DIRP.TO) -def nextentry(dirp): - """Read the next entry and returns an opaque object. - Use the methods has_xxx() and get_xxx() to read from that - opaque object. The opaque object is valid until the next - time nextentry() or closedir() is called. This may raise - OSError, or return a NULL pointer when exhausted. Note - that this doesn't filter out the "." and ".." entries. - """ - direntp = rposix.c_readdir(dirp) - if direntp: - error = rposix.get_saved_errno() - if error: - raise OSError(error, "readdir failed") - return direntp + def nextentry(dirp): + """Read the next entry and returns an opaque object. + Use the methods has_xxx() and get_xxx() to read from that + opaque object. The opaque object is valid until the next + time nextentry() or closedir() is called. This may raise + OSError, or return a NULL pointer when exhausted. Note + that this doesn't filter out the "." and ".." entries. + """ + direntp = rposix.c_readdir(dirp) + if direntp: + error = rposix.get_saved_errno() + if error: + raise OSError(error, "readdir failed") + return direntp -def has_name_bytes(direntp): - return True + def get_name_bytes(direntp): + namep = rffi.cast(rffi.CCHARP, direntp.c_d_name) + return rffi.charp2str(namep) -def get_name_bytes(direntp): - namep = rffi.cast(rffi.CCHARP, direntp.c_d_name) - return rffi.charp2str(namep) + DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0) + DT_REG = rposix.dirent_config.get('DT_REG', 255) + DT_DIR = rposix.dirent_config.get('DT_DIR', 255) + DT_LNK = rposix.dirent_config.get('DT_LNK', 255) -DT_UNKNOWN = rposix.dirent_config.get('DT_UNKNOWN', 0) -DT_REG = rposix.dirent_config.get('DT_REG', 255) -DT_DIR = rposix.dirent_config.get('DT_DIR', 255) -DT_LNK = rposix.dirent_config.get('DT_LNK', 255) + def get_known_type(direntp): + if rposix.HAVE_D_TYPE: + return rffi.getintfield(direntp, 'c_d_type') + return DT_UNKNOWN -def get_known_type(direntp): - if rposix.HAVE_D_TYPE: - return rffi.getintfield(direntp, 'c_d_type') - return DT_UNKNOWN + def get_inode(direntp): + return rffi.getintfield(direntp, 'c_d_ino') -def get_inode(direntp): - return rffi.getintfield(direntp, 'c_d_ino') +else: + # ----- Win32 version ----- + import stat + from rpython.rlib._os_support import unicode_traits + from rpython.rlib.rwin32file import make_win32_traits + from rpython.rlib import rposix_stat + + win32traits = make_win32_traits(unicode_traits) + + + SCANDIRP = lltype.Ptr(lltype.Struct('SCANDIRP', + ('filedata', win32traits.WIN32_FIND_DATA), + ('hFindFile', rwin32.HANDLE), + ('first_time', lltype.Bool), + )) + NULL_DIRP = lltype.nullptr(SCANDIRP.TO) + + + # must only be called with unicode! + def opendir(path): + if len(path) == 0: + path = u'.' + if path[-1] not in (u'\\', u'/', u':'): + mask = path + u'\\*.*' + else: + mask = path + u'*.*' + dirp = lltype.malloc(SCANDIRP.TO, flavor='raw') + hFindFile = win32traits.FindFirstFile(mask, dirp.filedata) + if hFindFile == rwin32.INVALID_HANDLE_VALUE: + error = rwin32.GetLastError_saved() + lltype.free(dirp, flavor='raw') + raise WindowsError(error, "FindFirstFileW failed") + dirp.hFindFile = hFindFile + dirp.first_time = True + return dirp + + def closedir(dirp): + if dirp.hFindFile != rwin32.INVALID_HANDLE_VALUE: + win32traits.FindClose(dirp.hFindFile) + lltype.free(dirp, flavor='raw') + + def nextentry(dirp): + """Read the next entry and returns an opaque object. + Use the methods has_xxx() and get_xxx() to read from that + opaque object. The opaque object is valid until the next + time nextentry() or closedir() is called. This may raise + WindowsError, or return NULL when exhausted. Note + that this doesn't filter out the "." and ".." entries. + """ + if dirp.first_time: + dirp.first_time = False + else: + if not win32traits.FindNextFile(dirp.hFindFile, dirp.filedata): + # error or no more files + error = rwin32.GetLastError_saved() + if error == win32traits.ERROR_NO_MORE_FILES: + return lltype.nullptr(win32traits.WIN32_FIND_DATA) + raise WindowsError(error, "FindNextFileW failed") + return dirp.filedata + + def get_name_unicode(filedata): + return unicode_traits.charp2str(rffi.cast(unicode_traits.CCHARP, + filedata.c_cFileName)) + + def get_known_type(filedata): + attr = filedata.c_dwFileAttributes + st_mode = rposix_stat.win32_attributes_to_mode(win32traits, attr) + return stat.S_IFMT(st_mode) + + def get_inode(filedata): + return None diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py --- a/rpython/rlib/rtime.py +++ b/rpython/rlib/rtime.py @@ -165,7 +165,7 @@ 'QueryPerformanceCounter', [rffi.CArrayPtr(lltype.SignedLongLong)], lltype.Void, releasegil=False) QueryPerformanceFrequency = external( - 'QueryPerformanceFrequency', [rffi.CArrayPtr(lltype.SignedLongLong)], + 'QueryPerformanceFrequency', [rffi.CArrayPtr(lltype.SignedLongLong)], rffi.INT, releasegil=False) class State(object): divisor = 0.0 @@ -267,9 +267,10 @@ else: void = lltype.nullptr(rffi.VOIDP.TO) with lltype.scoped_alloc(TIMEVAL) as t: - frac = math.fmod(secs, 1.0) + frac = int(math.fmod(secs, 1.0) * 1000000.) + assert frac >= 0 rffi.setintfield(t, 'c_tv_sec', int(secs)) - rffi.setintfield(t, 'c_tv_usec', int(frac*1000000.0)) + rffi.setintfield(t, 'c_tv_usec', frac) if rffi.cast(rffi.LONG, c_select(0, void, void, void, t)) != 0: errno = rposix.get_saved_errno() diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py --- a/rpython/rlib/rwin32file.py +++ b/rpython/rlib/rwin32file.py @@ -148,7 +148,7 @@ save_err=rffi.RFFI_SAVE_LASTERROR) FindClose = external('FindClose', [rwin32.HANDLE], - rwin32.BOOL) + rwin32.BOOL, releasegil=False) GetFileAttributes = external( 'GetFileAttributes' + suffix, diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,7 +47,7 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD -suffix = 'A' +suffix = 'W' RegSetValue = external( 'RegSetValue' + suffix, diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -1,3 +1,6 @@ +from hypothesis import given, strategies as st, assume +import pytest + from rpython.rtyper.test.test_llinterp import interpret from rpython.translator.c.test.test_genc import compile from rpython.tool.pytest.expecttest import ExpectTest @@ -8,10 +11,10 @@ import py def rposix_requires(funcname): - return py.test.mark.skipif(not hasattr(rposix, funcname), + return pytest.mark.skipif(not hasattr(rposix, funcname), reason="Requires rposix.%s()" % funcname) -win_only = py.test.mark.skipif("os.name != 'nt'") +win_only = pytest.mark.skipif("os.name != 'nt'") class TestPosixFunction: def test_access(self): @@ -827,3 +830,61 @@ rposix.lockf(fd, rposix.F_ULOCK, 4) finally: os.close(fd) + +def check_working_xattr(): + fname = str(udir.join('xattr_test0.txt')) + with open(fname, 'wb'): + pass + try: + rposix.setxattr(fname, 'user.foo', '') + except OSError: + return False + else: + return True + +@pytest.mark.skipif(not (hasattr(rposix, 'getxattr') and check_working_xattr()), + reason="Requires working rposix.getxattr()") +@given( + name=st.text( + alphabet=st.characters(min_codepoint=1), min_size=1, max_size=10), + value=st.binary(max_size=10), + follow_symlinks=st.booleans(), use_fd=st.booleans()) +def test_xattr(name, value, follow_symlinks, use_fd): + assume(follow_symlinks or not use_fd) + name = 'user.' + name.encode('utf-8') + fname = str(udir.join('xattr_test.txt')) + try: + os.unlink(fname) + except OSError: + pass + with open(fname, 'wb'): + pass + if use_fd: + file_id = os.open(fname, os.O_CREAT, 0777) + read, write, delete = rposix.fgetxattr, rposix.fsetxattr, rposix.fremovexattr + all_names = rposix.flistxattr + else: + file_id = fname + if follow_symlinks: + read, write, delete = rposix.getxattr, rposix.setxattr, rposix.removexattr + all_names = rposix.listxattr + else: + read = lambda *args, **kwargs: rposix.getxattr(*args, follow_symlinks=False, **kwargs) + write = lambda *args, **kwargs: rposix.setxattr(*args, follow_symlinks=False, **kwargs) + delete = lambda *args, **kwargs: rposix.removexattr(*args, follow_symlinks=False, **kwargs) + all_names = lambda *args, **kwargs: rposix.listxattr(*args, follow_symlinks=False, **kwargs) + try: + init_names = all_names(file_id) + with pytest.raises(OSError): + read(file_id, name) + write(file_id, name, value) + assert read(file_id, name) == value + assert set(all_names(file_id)) == set(init_names + [name]) + assert '' not in all_names(file_id) + delete(file_id, name) + with pytest.raises(OSError): + read(file_id, name) + assert set(all_names(file_id)) == set(init_names) + finally: + if use_fd: + os.close(file_id) _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit