Author: Richard Plangger <planri...@gmail.com> Branch: py3.5 Changeset: r88949:c10e210ecce7 Date: 2016-12-07 16:32 +0100 http://bitbucket.org/pypy/pypy/changeset/c10e210ecce7/
Log: merge py3.5-ssl diff --git a/lib-python/3/http/client.py b/lib-python/3/http/client.py --- a/lib-python/3/http/client.py +++ b/lib-python/3/http/client.py @@ -556,6 +556,7 @@ try: while True: chunk_left = self._get_chunk_left() + print("chunk_left", chunk_left) if chunk_left is None: break value.append(self._safe_read(chunk_left)) @@ -605,6 +606,7 @@ s = [] while amt > 0: chunk = self.fp.read(min(amt, MAXAMOUNT)) + print("read chunk %d %d", len(chunk), min(amt, MAXAMOUNT)) if not chunk: raise IncompleteRead(b''.join(s), amt) s.append(chunk) diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/crypto.py @@ -53,6 +53,7 @@ const char *OpenSSL_version(int); /* this is a macro in 1.1.0 */ +void *OPENSSL_malloc(size_t); void OPENSSL_free(void *); /* This was removed in 1.1.0 */ diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/evp.py @@ -25,6 +25,13 @@ static const int EVP_CTRL_GCM_GET_TAG; static const int EVP_CTRL_GCM_SET_TAG; +typedef struct { + int type; + int alias; + const char *name; + const char *data; +} OBJ_NAME; + static const int Cryptography_HAS_GCM; static const int Cryptography_HAS_PBKDF2_HMAC; static const int Cryptography_HAS_PKEY_CTX; @@ -136,6 +143,7 @@ without worrying about what OpenSSL we're running against. */ EVP_MD_CTX *Cryptography_EVP_MD_CTX_new(void); void Cryptography_EVP_MD_CTX_free(EVP_MD_CTX *); +void OBJ_NAME_do_all(int, void (*) (const OBJ_NAME *, void *), void *); """ MACROS = """ @@ -156,6 +164,7 @@ EC_KEY *EVP_PKEY_get1_EC_KEY(EVP_PKEY *); int EVP_PKEY_set1_EC_KEY(EVP_PKEY *, EC_KEY *); +int EVP_MD_CTX_block_size(const EVP_MD_CTX *md); int EVP_CIPHER_CTX_block_size(const EVP_CIPHER_CTX *); int EVP_CIPHER_CTX_ctrl(EVP_CIPHER_CTX *, int, int, void *); @@ -167,6 +176,7 @@ int EVP_PBE_scrypt(const char *, size_t, const unsigned char *, size_t, uint64_t, uint64_t, uint64_t, uint64_t, unsigned char *, size_t); +#define OBJ_NAME_TYPE_MD_METH ... """ CUSTOMIZATIONS = """ diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/ssl.py @@ -26,7 +26,6 @@ static const long Cryptography_HAS_SSL_CTX_SET_CLIENT_CERT_ENGINE; static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS; static const long Cryptography_HAS_NPN_NEGOTIATED; -static const long Cryptography_OPENSSL_NO_TLSEXT; /* Internally invented symbol to tell us if SNI is supported */ static const long Cryptography_HAS_TLSEXT_HOSTNAME; @@ -138,6 +137,7 @@ typedef ... SSL_CTX; typedef ... SSL_SESSION; + typedef ... SSL; static const long TLSEXT_NAMETYPE_host_name; @@ -434,7 +434,6 @@ long SSL_CTX_sess_misses(SSL_CTX *); long SSL_CTX_sess_timeouts(SSL_CTX *); long SSL_CTX_sess_cache_full(SSL_CTX *); - """ CUSTOMIZATIONS = """ @@ -689,12 +688,6 @@ static const long Cryptography_HAS_SSL_CTX_CLEAR_OPTIONS = 1; -#ifdef OPENSSL_NO_TLSEXT -static const long Cryptography_OPENSSL_NO_TLSEXT = 1; -#else -static const long Cryptography_OPENSSL_NO_TLSEXT = 0; -#endif - /* in OpenSSL 1.1.0 the SSL_ST values were renamed to TLS_ST and several were removed */ #if CRYPTOGRAPHY_OPENSSL_LESS_THAN_110 || defined(LIBRESSL_VERSION_NUMBER) diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py @@ -140,6 +140,7 @@ int X509_STORE_set_flags(X509_STORE *, unsigned long); void X509_STORE_free(X509_STORE *); + /* X509_STORE_CTX */ X509_STORE_CTX *X509_STORE_CTX_new(void); void X509_STORE_CTX_cleanup(X509_STORE_CTX *); @@ -201,7 +202,7 @@ int sk_X509_OBJECT_num(Cryptography_STACK_OF_X509_OBJECT *); X509_OBJECT *sk_X509_OBJECT_value(Cryptography_STACK_OF_X509_OBJECT *, int); -X509_VERIFY_PARAM * X509_STORE_get0_param(X509_STORE *); +X509_VERIFY_PARAM *X509_STORE_get0_param(X509_STORE *); Cryptography_STACK_OF_X509_OBJECT *X509_STORE_get0_objects(X509_STORE *); X509 *X509_OBJECT_get0_X509(X509_OBJECT *); int X509_OBJECT_get_type(const X509_OBJECT *); diff --git a/lib_pypy/_hashlib/__init__.py b/lib_pypy/_hashlib/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_hashlib/__init__.py @@ -0,0 +1,172 @@ +import sys +from threading import Lock +from _pypy_openssl import ffi, lib +from _cffi_ssl._stdssl.utility import (_str_to_ffi_buffer, _bytes_with_len, + _str_from_buf) + +def new(name, string=b''): + h = Hash(name) + h.update(string) + return h + +class Hash(object): + + def __init__(self, name, copy_from=None): + self.ctx = ffi.NULL + self.name = name + digest_type = self.digest_type_by_name() + self.digest_size = lib.EVP_MD_size(digest_type) + + # Allocate a lock for each HASH object. + # An optimization would be to not release the GIL on small requests, + # and use a custom lock only when needed. + self.lock = Lock() + + ctx = lib.Cryptography_EVP_MD_CTX_new() + if ctx == ffi.NULL: + raise MemoryError + ctx = ffi.gc(ctx, lib.Cryptography_EVP_MD_CTX_free) + + try: + if copy_from is not None: + # cpython uses EVP_MD_CTX_copy(...) + if not lib.EVP_MD_CTX_copy_ex(ctx, copy_from): + raise ValueError + else: + # cpython uses EVP_DigestInit + lib.EVP_DigestInit_ex(ctx, digest_type, ffi.NULL) + self.ctx = ctx + except: + # no need to gc ctx! + raise + + def digest_type_by_name(self): + c_name = _str_to_ffi_buffer(self.name) + digest_type = lib.EVP_get_digestbyname(c_name) + if not digest_type: + raise ValueError("unknown hash function") + # TODO + return digest_type + + def __repr__(self): + return "<%s HASH object at 0x%s>" % (self.name, id(self)) + + def update(self, string): + buf = ffi.from_buffer(string) + with self.lock: + # XXX try to not release the GIL for small requests + lib.EVP_DigestUpdate(self.ctx, buf, len(buf)) + + def copy(self): + """Return a copy of the hash object.""" + with self.lock: + return Hash(self.name, copy_from=self.ctx) + + def digest(self): + """Return the digest value as a string of binary data.""" + return self._digest() + + def hexdigest(self): + """Return the digest value as a string of hexadecimal digits.""" + digest = self._digest() + hexdigits = '0123456789abcdef' + result = [] + for c in digest: + result.append(hexdigits[(c >> 4) & 0xf]) + result.append(hexdigits[ c & 0xf]) + return ''.join(result) + + @property + def block_size(self): + return lib.EVP_MD_CTX_block_size(self.ctx) + + def _digest(self): + ctx = lib.Cryptography_EVP_MD_CTX_new() + if ctx == ffi.NULL: + raise MemoryError + try: + with self.lock: + if not lib.EVP_MD_CTX_copy_ex(ctx, self.ctx): + raise ValueError + digest_size = self.digest_size + buf = ffi.new("unsigned char[]", digest_size) + lib.EVP_DigestFinal_ex(ctx, buf, ffi.NULL) + return _bytes_with_len(buf, digest_size) + finally: + lib.Cryptography_EVP_MD_CTX_free(ctx) + +algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') + +class NameFetcher: + def __init__(self): + self.meth_names = [] + self.error = None + + +def _fetch_names(): + name_fetcher = NameFetcher() + handle = ffi.new_handle(name_fetcher) + lib.OBJ_NAME_do_all(lib.OBJ_NAME_TYPE_MD_METH, hash_name_mapper_callback, handle) + if name_fetcher.error: + raise name_fetcher.error + meth_names = name_fetcher.meth_names + name_fetcher.meth_names = None + return frozenset(meth_names) + +@ffi.callback("void(OBJ_NAME*, void*)") +def hash_name_mapper_callback(obj_name, userdata): + if not obj_name: + return + name_fetcher = ffi.from_handle(userdata) + # Ignore aliased names, they pollute the list and OpenSSL appears + # to have a its own definition of alias as the resulting list + # still contains duplicate and alternate names for several + # algorithms. + if obj_name.alias != 0: + return + name = _str_from_buf(obj_name.name) + name_fetcher.meth_names.append(name) + +openssl_md_meth_names = _fetch_names() +del _fetch_names + +# shortcut functions +def make_new_hash(name, funcname): + def new_hash(string=b''): + return new(name, string) + new_hash.__name__ = funcname + return new_hash + +for _name in algorithms: + _newname = 'openssl_%s' % (_name,) + globals()[_newname] = make_new_hash(_name, _newname) + +if hasattr(lib, 'PKCS5_PBKDF2_HMAC'): + #@unwrap_spec(name=str, password='bytes', salt='bytes', iterations=int, + # w_dklen=WrappedDefault(None)) + def pbkdf2_hmac(name, password, salt, iterations, dklen=None): + if not isinstance(name, str): + raise TypeError("expected 'str' for name, but got %s" % type(name)) + c_name = _str_to_ffi_buffer(name) + digest = lib.EVP_get_digestbyname(c_name) + if digest == ffi.NULL: + raise ValueError("unsupported hash type") + if dklen is None: + dklen = lib.EVP_MD_size(digest) + if dklen < 1: + raise ValueError("key length must be greater than 0.") + if dklen >= sys.maxsize: + raise OverflowError("key length is too great.") + if iterations < 1: + raise ValueError("iteration value must be greater than 0.") + if iterations >= sys.maxsize: + raise OverflowError("iteration value is too great.") + buf = ffi.new("unsigned char[]", dklen) + c_password = ffi.from_buffer(bytes(password)) + c_salt = ffi.from_buffer(bytes(salt)) + r = lib.PKCS5_PBKDF2_HMAC(c_password, len(c_password), + ffi.cast("unsigned char*",c_salt), len(c_salt), + iterations, digest, dklen, buf) + if r == 0: + raise ValueError + return _bytes_with_len(buf, dklen) diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -149,10 +149,11 @@ def connect(database, timeout=5.0, detect_types=0, isolation_level="", - check_same_thread=True, factory=None, cached_statements=100): + check_same_thread=True, factory=None, cached_statements=100, + uri=0): factory = Connection if not factory else factory return factory(database, timeout, detect_types, isolation_level, - check_same_thread, factory, cached_statements) + check_same_thread, factory, cached_statements, uri) def _unicode_text_factory(x): @@ -195,14 +196,23 @@ _db = None def __init__(self, database, timeout=5.0, detect_types=0, isolation_level="", - check_same_thread=True, factory=None, cached_statements=100): + check_same_thread=True, factory=None, cached_statements=100, uri=0): self.__initialized = True db_star = _ffi.new('sqlite3 **') if isinstance(database, unicode): database = database.encode('utf-8') - if _lib.sqlite3_open(database, db_star) != _lib.SQLITE_OK: - raise OperationalError("Could not open database") + if _lib.SQLITE_OPEN_URI != 0: + if uri and _lib.SQLITE_OPEN_URI == 0: + raise NotSupportedError("URIs not supported") + flags = _lib.SQLITE_OPEN_READWRITE | _lib.SQLITE_OPEN_CREATE + if uri: + flags |= _lib.SQLITE_OPEN_URI + if _lib.sqlite3_open_v2(database, db_star, flags, _ffi.NULL) != _lib.SQLITE_OK: + raise OperationalError("Could not open database") + else: + if _lib.sqlite3_open(database, db_star) != _lib.SQLITE_OK: + raise OperationalError("Could not open database") self._db = db_star[0] if timeout is not None: timeout = int(timeout * 1000) # pysqlite2 uses timeout in seconds @@ -1195,6 +1205,8 @@ def __getitem__(self, item): if isinstance(item, (int, long)): return self.values[item] + elif isinstance(item, slice): + return self.values[item] else: item = item.lower() for idx, desc in enumerate(self.description): diff --git a/lib_pypy/_sqlite3_build.py b/lib_pypy/_sqlite3_build.py --- a/lib_pypy/_sqlite3_build.py +++ b/lib_pypy/_sqlite3_build.py @@ -103,6 +103,10 @@ #define SQLITE_DROP_VTABLE ... #define SQLITE_FUNCTION ... +static const long SQLITE_OPEN_URI; +static const long SQLITE_OPEN_READWRITE; +static const long SQLITE_OPEN_CREATE; + const char *sqlite3_libversion(void); typedef ... sqlite3; @@ -117,6 +121,13 @@ sqlite3 **ppDb /* OUT: SQLite db handle */ ); +int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +); + int sqlite3_close(sqlite3 *); int sqlite3_busy_timeout(sqlite3*, int ms); @@ -259,7 +270,21 @@ libraries=['sqlite3'] ) -_ffi.set_source("_sqlite3_cffi", "#include <sqlite3.h>", **extra_args) +SOURCE = """ +#include <sqlite3.h> + +#ifndef SQLITE_OPEN_URI +static const long SQLITE_OPEN_URI = 0; +#endif +#ifndef SQLITE_OPEN_READWRITE +static const long SQLITE_OPEN_READWRITE = 0; +#endif +#ifndef SQLITE_OPEN_CREATE +static const long SQLITE_OPEN_CREATE = 0; +#endif +""" + +_ffi.set_source("_sqlite3_cffi", SOURCE, **extra_args) if __name__ == "__main__": diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -35,7 +35,7 @@ working_modules.update([ "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", - "zlib", "bz2", "struct", "_hashlib", "_md5", "_minimal_curses", + "zlib", "bz2", "struct", "_md5", "_minimal_curses", "thread", "itertools", "pyexpat", "cpyext", "array", "binascii", "_multiprocessing", '_warnings', "_collections", "_multibytecodec", "_continuation", "_cffi_backend", @@ -118,7 +118,6 @@ "zlib" : ["rpython.rlib.rzlib"], "bz2" : ["pypy.module.bz2.interp_bz2"], "pyexpat" : ["pypy.module.pyexpat.interp_pyexpat"], - "_hashlib" : ["pypy.module._ssl.interp_ssl"], "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], "_continuation": ["rpython.rlib.rstacklet"], "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py --- a/pypy/interpreter/test/test_appinterp.py +++ b/pypy/interpreter/test/test_appinterp.py @@ -156,7 +156,7 @@ assert space1.str_w(w_str) == "hello" class TestMixedModuleUnfreeze: - spaceconfig = dict(usemodules=('_ssl', '_socket')) + spaceconfig = dict(usemodules=('_socket',)) def test_random_stuff_can_unfreeze(self): # When a module contains an "import" statement in applevel code, the @@ -167,13 +167,13 @@ # at runtime, like setting os.environ (posix module) or initializing # the winsock library (_socket module) w_socket = self.space.builtin_modules['_socket'] - w_ssl = self.space.builtin_modules['_ssl'] + # _ssl is not builtin anymore, this test also tried to _cleanup_ on + # the wrapped ssl object + # w_ssl = self.space.builtin_modules['_ssl'] # Uncomment this line for a workaround # space.getattr(w_ssl, space.wrap('SSLError')) w_socket._cleanup_() assert w_socket.startup_called == False - w_ssl._cleanup_() # w_ssl.appleveldefs['SSLError'] imports _socket - assert w_socket.startup_called == False diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py deleted file mode 100644 --- a/pypy/module/_hashlib/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -from pypy.interpreter.mixedmodule import MixedModule -from pypy.module._hashlib.interp_hashlib import ( - algorithms, fetch_names, HAS_FAST_PKCS5_PBKDF2_HMAC) - - -class Module(MixedModule): - interpleveldefs = { - 'new' : 'interp_hashlib.new', - } - - appleveldefs = { - } - - for name in algorithms: - interpleveldefs['openssl_' + name] = 'interp_hashlib.new_' + name - - if HAS_FAST_PKCS5_PBKDF2_HMAC: - interpleveldefs['pbkdf2_hmac'] = 'interp_hashlib.pbkdf2_hmac' - - def startup(self, space): - w_meth_names = fetch_names(space) - space.setattr(self, space.wrap('openssl_md_meth_names'), w_meth_names) diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py deleted file mode 100644 --- a/pypy/module/_hashlib/interp_hashlib.py +++ /dev/null @@ -1,209 +0,0 @@ -from __future__ import with_statement - -from rpython.rlib import rgc, ropenssl -from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib.rstring import StringBuilder -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.tool.sourcetools import func_renamer - -from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.error import OperationError, oefmt -from pypy.interpreter.gateway import unwrap_spec, interp2app, WrappedDefault -from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.thread.os_lock import Lock - - -algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') - -def hash_name_mapper_callback(obj_name, userdata): - if not obj_name: - return - # Ignore aliased names, they pollute the list and OpenSSL appears - # to have a its own definition of alias as the resulting list - # still contains duplicate and alternate names for several - # algorithms. - if rffi.cast(lltype.Signed, obj_name[0].c_alias): - return - try: - space = global_name_fetcher.space - w_name = space.wrap(rffi.charp2str(obj_name[0].c_name)) - global_name_fetcher.meth_names.append(w_name) - except OperationError as e: - global_name_fetcher.w_error = e - -class NameFetcher: - def setup(self, space): - self.space = space - self.meth_names = [] - self.w_error = None - def _cleanup_(self): - self.__dict__.clear() -global_name_fetcher = NameFetcher() - -def fetch_names(space): - global_name_fetcher.setup(space) - ropenssl.init_digests() - ropenssl.OBJ_NAME_do_all(ropenssl.OBJ_NAME_TYPE_MD_METH, - hash_name_mapper_callback, None) - if global_name_fetcher.w_error: - raise global_name_fetcher.w_error - meth_names = global_name_fetcher.meth_names - global_name_fetcher.meth_names = None - return space.call_function(space.w_frozenset, space.newlist(meth_names)) - -class W_Hash(W_Root): - NULL_CTX = lltype.nullptr(ropenssl.EVP_MD_CTX.TO) - ctx = NULL_CTX - - def __init__(self, space, name, copy_from=NULL_CTX): - self.name = name - digest_type = self.digest_type_by_name(space) - self.digest_size = ropenssl.EVP_MD_size(digest_type) - - # Allocate a lock for each HASH object. - # An optimization would be to not release the GIL on small requests, - # and use a custom lock only when needed. - self.lock = Lock(space) - - ctx = ropenssl.EVP_MD_CTX_new() - if ctx is None: - raise MemoryError - rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size) - try: - if copy_from: - if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from): - raise ValueError - else: - ropenssl.EVP_DigestInit(ctx, digest_type) - self.ctx = ctx - except: - ropenssl.EVP_MD_CTX_free(ctx) - raise - self.register_finalizer(space) - - def _finalize_(self): - ctx = self.ctx - if ctx: - self.ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO) - ropenssl.EVP_MD_CTX_free(ctx) - - def digest_type_by_name(self, space): - digest_type = ropenssl.EVP_get_digestbyname(self.name) - if not digest_type: - raise oefmt(space.w_ValueError, "unknown hash function") - return digest_type - - def descr_repr(self, space): - addrstring = self.getaddrstring(space) - return space.wrap("<%s HASH object at 0x%s>" % ( - self.name, addrstring)) - - @unwrap_spec(string='bufferstr') - def update(self, space, string): - with rffi.scoped_nonmovingbuffer(string) as buf: - with self.lock: - # XXX try to not release the GIL for small requests - ropenssl.EVP_DigestUpdate(self.ctx, buf, len(string)) - - def copy(self, space): - "Return a copy of the hash object." - with self.lock: - return W_Hash(space, self.name, copy_from=self.ctx) - - def digest(self, space): - "Return the digest value as a string of binary data." - digest = self._digest(space) - return space.newbytes(digest) - - def hexdigest(self, space): - "Return the digest value as a string of hexadecimal digits." - digest = self._digest(space) - hexdigits = '0123456789abcdef' - result = StringBuilder(self.digest_size * 2) - for c in digest: - result.append(hexdigits[(ord(c) >> 4) & 0xf]) - result.append(hexdigits[ ord(c) & 0xf]) - return space.wrap(result.build()) - - def get_digest_size(self, space): - return space.wrap(self.digest_size) - - def get_block_size(self, space): - digest_type = self.digest_type_by_name(space) - block_size = ropenssl.EVP_MD_block_size(digest_type) - return space.wrap(block_size) - - def get_name(self, space): - return space.wrap(self.name) - - def _digest(self, space): - ctx = ropenssl.EVP_MD_CTX_new() - if ctx is None: - raise MemoryError - try: - with self.lock: - if not ropenssl.EVP_MD_CTX_copy(ctx, self.ctx): - raise ValueError - digest_size = self.digest_size - with rffi.scoped_alloc_buffer(digest_size) as buf: - ropenssl.EVP_DigestFinal(ctx, buf.raw, None) - return buf.str(digest_size) - finally: - ropenssl.EVP_MD_CTX_free(ctx) - - -W_Hash.typedef = TypeDef( - 'HASH', - __repr__=interp2app(W_Hash.descr_repr), - update=interp2app(W_Hash.update), - copy=interp2app(W_Hash.copy), - digest=interp2app(W_Hash.digest), - hexdigest=interp2app(W_Hash.hexdigest), - # - digest_size=GetSetProperty(W_Hash.get_digest_size), - block_size=GetSetProperty(W_Hash.get_block_size), - name=GetSetProperty(W_Hash.get_name), -) -W_Hash.typedef.acceptable_as_base_class = False - -@unwrap_spec(name=str, string='bufferstr') -def new(space, name, string=''): - w_hash = W_Hash(space, name) - w_hash.update(space, string) - return space.wrap(w_hash) - -# shortcut functions -def make_new_hash(name, funcname): - @func_renamer(funcname) - @unwrap_spec(string='bufferstr') - def new_hash(space, string=''): - return new(space, name, string) - return new_hash - -for _name in algorithms: - _newname = 'new_%s' % (_name,) - globals()[_newname] = make_new_hash(_name, _newname) - - -HAS_FAST_PKCS5_PBKDF2_HMAC = ropenssl.PKCS5_PBKDF2_HMAC is not None -if HAS_FAST_PKCS5_PBKDF2_HMAC: - @unwrap_spec(name=str, password='bytes', salt='bytes', rounds=int, - w_dklen=WrappedDefault(None)) - def pbkdf2_hmac(space, name, password, salt, rounds, w_dklen): - digest = ropenssl.EVP_get_digestbyname(name) - if not digest: - raise oefmt(space.w_ValueError, "unknown hash function") - if space.is_w(w_dklen, space.w_None): - dklen = ropenssl.EVP_MD_size(digest) - else: - dklen = space.int_w(w_dklen) - if dklen < 1: - raise oefmt(space.w_ValueError, - "key length must be greater than 0.") - with rffi.scoped_alloc_buffer(dklen) as buf: - r = ropenssl.PKCS5_PBKDF2_HMAC( - password, len(password), salt, len(salt), rounds, digest, - dklen, buf.raw) - if not r: - raise ValueError - return space.newbytes(buf.str(dklen)) diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py deleted file mode 100644 --- a/pypy/module/_hashlib/test/test_hashlib.py +++ /dev/null @@ -1,100 +0,0 @@ -class AppTestHashlib: - spaceconfig = { - "usemodules": ['_hashlib', 'array', 'struct', 'binascii'], - } - - def test_method_names(self): - import _hashlib - assert isinstance(_hashlib.openssl_md_meth_names, frozenset) - assert "md5" in _hashlib.openssl_md_meth_names - - def test_simple(self): - import _hashlib - assert _hashlib.new('md5').__class__.__name__ == 'HASH' - assert len(_hashlib.new('md5').hexdigest()) == 32 - - def test_attributes(self): - import hashlib - for name, (expected_size, expected_block_size) in { - 'md5': (16, 64), - 'sha1': (20, 64), - 'sha224': (28, 64), - 'sha256': (32, 64), - 'sha384': (48, 128), - 'sha512': (64, 128), - }.items(): - h = hashlib.new(name) - assert h.name == name - assert h.digest_size == expected_size - assert h.block_size == expected_block_size - # - h.update(b'abc') - h2 = h.copy() - h.update(b'def') - digest = h.digest() - hexdigest = h.hexdigest() - h2.update(b'd') - h2.update(b'ef') - assert digest == h2.digest() - assert hexdigest == h2.hexdigest() - assert len(digest) == h.digest_size - assert len(hexdigest) == h.digest_size * 2 - c_digest = digest - c_hexdigest = hexdigest - - - def test_shortcut(self): - import hashlib - assert repr(hashlib.md5()).startswith("<md5 HASH object") - - def test_uppercase(self): - import _hashlib - h = _hashlib.new('MD5') - assert h.digest_size == 16 - assert len(h.hexdigest()) == 32 - - def test_buffer(self): - import _hashlib, array - b = array.array('b', b'x' * 10) - h = _hashlib.new('md5', b) - h.update(b) - assert h.digest() == _hashlib.openssl_md5(b'x' * 20).digest() - _hashlib.openssl_sha1(b).digest() - - def test_extra_algorithms(self): - expected_results = { - "md5": "bb649c83dd1ea5c9d9dec9a18df0ffe9", - "md4": "c275b8454684ea416b93d7a418b43176", - "mdc2": None, # XXX find the correct expected value - "sha": "e2b0a8609b47c58e5d984c9ccfe69f9b654b032b", - "ripemd160": "cc4a5ce1b3df48aec5d22d1f16b894a0b894eccc", - "whirlpool": ("1a22b79fe5afda02c63a25927193ed01dc718b74" - "026e597608ce431f9c3d2c9e74a7350b7fbb7c5d" - "4effe5d7a31879b8b7a10fd2f544c4ca268ecc6793923583"), - } - import _hashlib - test_string = b"Nobody inspects the spammish repetition" - for hash_name, expected in sorted(expected_results.items()): - try: - m = _hashlib.new(hash_name) - except ValueError as e: - print('skipped %s: %s' % (hash_name, e)) - continue - m.update(test_string) - got = m.hexdigest() - assert got and type(got) is str and len(got) % 2 == 0 - if expected is not None: - assert got == expected - - def test_pbkdf2(self): - try: - from _hashlib import pbkdf2_hmac - except ImportError: - skip("Requires OpenSSL >= 1.1") - out = pbkdf2_hmac('sha1', b'password', b'salt', 1) - assert type(out) is bytes - assert out == '0c60c80f961f0e71f3a9b524af6012062fe037a6'.decode('hex') - out = pbkdf2_hmac('sha1', b'password', b'salt', 2, None) - assert out == 'ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'.decode('hex') - raises(TypeError, pbkdf2_hmac, 'sha1', 'password', b'salt', 1) - raises(TypeError, pbkdf2_hmac, 'sha1', b'password', 'salt', 1) diff --git a/pypy/module/_hashlib/test/test_ztranslation.py b/pypy/module/_hashlib/test/test_ztranslation.py deleted file mode 100644 --- a/pypy/module/_hashlib/test/test_ztranslation.py +++ /dev/null @@ -1,4 +0,0 @@ -from pypy.objspace.fake.checkmodule import checkmodule - -def test_checkmodule(): - checkmodule('_hashlib') diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -534,7 +534,9 @@ """ rwbuffer = space.getarg_w('w*', w_buffer) lgt = rwbuffer.getlength() - if nbytes == 0 or nbytes > lgt: + if nbytes < 0: + raise oefmt(space.w_ValueError, "negative buffersize in recv_into") + if nbytes == 0: nbytes = lgt while True: try: diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -868,6 +868,22 @@ posix.close(fileno) cli.close() + def test_recv_into_params(self): + import os + import _socket + cli = _socket.socket() + cli.connect(self.serv.getsockname()) + fileno, addr = self.serv._accept() + os.write(fileno, b"abcdef") + # + m = memoryview(bytearray(5)) + raises(ValueError, cli.recv_into, m, -1) + raises(ValueError, cli.recv_into, m, 6) + cli.recv_into(m,5) + assert m.tobytes() == b"abcde" + os.close(fileno) + cli.close() + class AppTestErrno: spaceconfig = {'usemodules': ['_socket', 'select']} diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py --- a/pypy/module/struct/formatiterator.py +++ b/pypy/module/struct/formatiterator.py @@ -109,6 +109,7 @@ self.buf = buf self.length = buf.getlength() self.pos = 0 + self.strides = None self.result_w = [] # list of wrapped objects # See above comment on operate. @@ -126,10 +127,15 @@ self.pos = (self.pos + mask) & ~mask def finished(self): - if self.pos != self.length: + value = self.pos + if self.strides and self.strides[0] < 0: + value = -self.pos + if value != self.length: raise StructError("unpack str size too long for format") def read(self, count): + if self.strides: + count = self.strides[0] end = self.pos + count if end > self.length: raise StructError("unpack str size too short for format") @@ -151,5 +157,14 @@ string, pos = self.buf.as_str_and_offset_maybe() return string, pos+self.pos - def skip(self, size): - self.read(size) # XXX, could avoid taking the slice + def skip(self, count): + # assumption: UnpackFormatIterator only iterates over + # flat structures (continous memory) either: forward (index + # grows) or reverse + if self.strides: + assert len(self.strides) == 1 + count = self.strides[0] + end = self.pos + count + if end > self.length: + raise StructError("unpack str size too short for format") + self.pos = end diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -115,8 +115,6 @@ return ''.join(self.copy_buffer()) def copy_buffer(self): - buf = self.buf - n_bytes = buf.getlength() data = [] self._copy_rec(0, data, 0) return data @@ -130,7 +128,6 @@ self._copy_base(data,off) return - # TODO add a test that has at least 2 dims for i in range(shape): self._copy_rec(idim+1,data,off) off += strides[idim] @@ -140,18 +137,21 @@ step = shapes[0] strides = self.getstrides() itemsize = self.getitemsize() + bytesize = self.getlength() + copiedbytes = 0 for i in range(step): bytes = self.buf.getslice(off, off+itemsize, 1, itemsize) data.append(bytes) + copiedbytes += len(bytes) off += strides[0] # do notcopy data if the sub buffer is out of bounds - if off >= self.buf.getlength(): + if copiedbytes >= bytesize: break def getlength(self): if self.length != -1: - return self.length // self.itemsize - return self.buf.getlength() // self.itemsize + return self.length + return self.buf.getlength() def descr_tobytes(self, space): self._check_released(space) @@ -167,13 +167,20 @@ raise NotImplementedError elif dim == 1: itemsize = self.getitemsize() - return self._tolist(space, buf, buf.getlength() // itemsize, fmt) + return self._tolist(space, buf, self.getlength(), itemsize, fmt, + self.getstrides()) else: return self._tolist_rec(space, buf, 0, 0, fmt) - def _tolist(self, space, buf, count, fmt): + def _tolist(self, space, buf, bytecount, itemsize, fmt, strides=None): # TODO: this probably isn't very fast + count = bytecount // itemsize fmtiter = UnpackFormatIterator(space, buf) + # patch the length, necessary buffer might have offset + # which leads to wrong length calculation if e.g. the + # memoryview is reversed + fmtiter.length = bytecount + fmtiter.strides = strides fmtiter.interpret(fmt * count) return space.newlist(fmtiter.result_w) @@ -188,12 +195,13 @@ # if dim >= self.getndim(): bytecount = (stride * dimshape) - count = bytecount // itemsize - return self._tolist(space, buf, count, fmt) + return self._tolist(space, buf, bytecount, itemsize, fmt, [stride]) items = [None] * dimshape + orig_buf = buf for i in range(dimshape): - item = self._tolist_rec(space, SubBuffer(buf, start, stride), start, idim+1, fmt) + buf = SubBuffer(orig_buf, start, stride) + item = self._tolist_rec(space, buf, start, idim+1, fmt) items[i] = item start += stride @@ -212,23 +220,21 @@ while dim < length: w_obj = w_tuple.getitem(space, dim) index = space.getindex_w(w_obj, space.w_IndexError) - start = self.lookup_dimension(space, start, dim, index) + shape = self.buf.getshape() + strides = self.buf.getstrides() + start = self.lookup_dimension(space, shape, strides, start, dim, index) dim += 1 return start - def lookup_dimension(self, space, start, dim, index): - view = self.buf - shape = view.getshape() - strides = view.getstrides() + def lookup_dimension(self, space, shape, strides, start, dim, index): nitems = shape[dim] if index < 0: index += nitems if index < 0 or index >= nitems: raise oefmt(space.w_IndexError, "index out of bounds on dimension %d", dim+1) - start += strides[dim] * index # TODO suboffsets? - return start + return start + strides[dim] * index def _getitem_tuple_indexed(self, space, w_index): view = self.buf @@ -253,50 +259,65 @@ fmtiter.interpret(fmt) return fmtiter.result_w[0] + def _decode_index(self, space, w_index, is_slice): + shape = self.getshape() + if len(shape) == 0: + count = 1 + else: + count = shape[0] + return space.decode_index4(w_index, count) def descr_getitem(self, space, w_index): self._check_released(space) if space.isinstance_w(w_index, space.w_tuple): return self._getitem_tuple_indexed(space, w_index) - - start, stop, step, size = space.decode_index4(w_index, self.getlength()) + is_slice = space.isinstance_w(w_index, space.w_slice) + start, stop, step, slicelength = self._decode_index(space, w_index, is_slice) # ^^^ for a non-slice index, this returns (index, 0, 0, 1) if step == 0: # index only itemsize = self.getitemsize() - if itemsize == 1: - ch = self.buf.getitem(start) - return space.newint(ord(ch)) + dim = self.getndim() + if dim == 0: + raise oefmt(space.w_TypeError, "invalid indexing of 0-dim memory") + elif dim == 1: + shape = self.getshape() + strides = self.getstrides() + idx = self.lookup_dimension(space, shape, strides, 0, 0, start) + if itemsize == 1: + ch = self.buf.getitem(idx) + return space.newint(ord(ch)) + else: + # TODO: this probably isn't very fast + buf = SubBuffer(self.buf, idx, itemsize) + fmtiter = UnpackFormatIterator(space, buf) + fmtiter.length = buf.getlength() + fmtiter.interpret(self.format) + return fmtiter.result_w[0] else: - # TODO: this probably isn't very fast - buf = SubBuffer(self.buf, start*itemsize, itemsize) - fmtiter = UnpackFormatIterator(space, buf) - fmtiter.interpret(self.format) - return fmtiter.result_w[0] - elif step == 1: + raise oefmt(space.w_NotImplementedError, "multi-dimensional sub-views are not implemented") + elif is_slice: mv = W_MemoryView.copy(self) - mv.slice(start, step, size) + mv.init_slice(start, stop, step, slicelength, 0) + mv.init_len() mv._init_flags() return mv + # multi index is handled at the top of this function else: - mv = W_MemoryView.copy(self) - mv.slice(start, step, size) - mv.length = mv.bytecount_from_shape() - mv._init_flags() - return mv + raise TypeError("memoryview: invalid slice key") - def slice(self, start, step, size): + def init_slice(self, start, stop, step, slicelength, dim): # modifies the buffer, shape and stride to allow step to be > 1 + self.strides = strides = self.getstrides()[:] + self.shape = shape = self.getshape()[:] + bytesize = self.getitemsize() * slicelength + self.buf = SubBuffer(self.buf, strides[dim] * start, bytesize) + shape[dim] = slicelength + strides[dim] = strides[dim] * step # TODO subbuffer - strides = self.getstrides()[:] - shape = self.getshape()[:] - itemsize = self.getitemsize() - dim = 0 - self.buf = SubBuffer(self.buf, strides[dim] * start, size*step*itemsize) - shape[dim] = size - strides[dim] = strides[dim] * step - self.strides = strides - self.shape = shape + + def init_len(self): + self.length = self.bytecount_from_shape() def bytecount_from_shape(self): dim = self.getndim() @@ -307,10 +328,9 @@ return length * self.getitemsize() @staticmethod - def copy(view, buf=None): + def copy(view): # TODO suboffsets - if buf == None: - buf = view.buf + buf = view.buf return W_MemoryView(buf, view.getformat(), view.getitemsize(), view.getndim(), view.getshape()[:], view.getstrides()[:]) @@ -321,11 +341,16 @@ if space.isinstance_w(w_index, space.w_tuple): raise oefmt(space.w_NotImplementedError, "") start, stop, step, size = space.decode_index4(w_index, self.getlength()) + is_slice = space.isinstance_w(w_index, space.w_slice) + start, stop, step, slicelength = self._decode_index(space, w_index, is_slice) itemsize = self.getitemsize() if step == 0: # index only + shape = self.getshape() + strides = self.getstrides() + idx = self.lookup_dimension(space, shape, strides, 0, 0, start) if itemsize == 1: ch = getbytevalue(space, w_obj) - self.buf.setitem(start, ch) + self.buf.setitem(idx, ch) else: # TODO: this probably isn't very fast fmtiter = PackFormatIterator(space, [w_obj], itemsize) @@ -335,10 +360,10 @@ raise oefmt(space.w_TypeError, "memoryview: invalid type for format '%s'", self.format) - self.buf.setslice(start * itemsize, fmtiter.result.build()) + self.buf.setslice(idx, fmtiter.result.build()) elif step == 1: value = space.buffer_w(w_obj, space.BUF_CONTIG_RO) - if value.getlength() != size * itemsize: + if value.getlength() != slicelength * itemsize: raise oefmt(space.w_ValueError, "cannot modify size of memoryview object") self.buf.setslice(start * itemsize, value.as_str()) @@ -354,11 +379,11 @@ src = space.buffer_w(w_obj, space.BUF_CONTIG_RO) dst_strides = self.getstrides() dim = 0 - dst = SubBuffer(self.buf, start * itemsize, size * itemsize) + dst = SubBuffer(self.buf, start * itemsize, slicelength * itemsize) src_stride0 = dst_strides[dim] off = 0 - src_shape0 = size + src_shape0 = slicelength src_stride0 = src.getstrides()[0] if isinstance(w_obj, W_MemoryView): src_stride0 = w_obj.getstrides()[0] @@ -373,11 +398,15 @@ def descr_len(self, space): self._check_released(space) - return space.wrap(self.getlength()) + dim = self.getndim() + if dim == 0: + return space.newint(1) + shape = self.getshape() + return space.wrap(shape[0]) def w_get_nbytes(self, space): self._check_released(space) - return space.wrap(self.buf.getlength()) + return space.wrap(self.getlength()) def w_get_format(self, space): self._check_released(space) @@ -385,11 +414,11 @@ def w_get_itemsize(self, space): self._check_released(space) - return space.wrap(self.itemsize) + return space.wrap(self.getitemsize()) def w_get_ndim(self, space): self._check_released(space) - return space.wrap(self.buf.getndim()) + return space.wrap(self.getndim()) def w_is_readonly(self, space): self._check_released(space) @@ -397,13 +426,13 @@ def w_get_shape(self, space): self._check_released(space) - if self.buf.getndim() == 0: + if self.getndim() == 0: return space.w_None return space.newtuple([space.wrap(x) for x in self.getshape()]) def w_get_strides(self, space): self._check_released(space) - if self.buf.getndim() == 0: + if self.getndim() == 0: return space.w_None return space.newtuple([space.wrap(x) for x in self.getstrides()]) @@ -615,8 +644,6 @@ return None def _cast_to_ND(self, space, shape, ndim): - buf = self.buf - self.ndim = ndim length = self.itemsize if ndim == 0: diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -1,4 +1,5 @@ import py +import pytest import struct from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import interp2app @@ -42,13 +43,13 @@ def test_extended_slice(self): data = bytearray(b'abcefg') v = memoryview(data) - w = v[0:2:2] # failing for now: NotImplementedError + w = v[0:2:2] assert len(w) == 1 assert list(w) == [97] v[::2] = b'ABC' assert data == bytearray(eval("b'AbBeCg'")) - assert v[::2] == b'ABC' - assert v[::-2] == b'geb' + assert v[::2].tobytes() == b'ABC' + assert v[::-2].tobytes() == b'geb' def test_memoryview_attrs(self): v = memoryview(b"a"*100) @@ -409,3 +410,33 @@ v = view.cast('h', shape=(3,2)) assert v.tolist() == [[2,3],[4,5],[6,7]] raises(TypeError, "view.cast('h', shape=(3,3))") + + def test_reversed(self): + bytes = b"\x01\x01\x02\x02\x03\x03" + view = memoryview(bytes) + revlist = list(reversed(view.tolist())) + assert view[::-1][0] == 3 + assert view[::-1][1] == 3 + assert view[::-1][2] == 2 + assert view[::-1][3] == 2 + assert view[::-1][4] == 1 + assert view[::-1][5] == 1 + assert view[::-1][-1] == 1 + assert view[::-1][-2] == 1 + assert list(reversed(view)) == revlist + assert list(reversed(view)) == view[::-1].tolist() + +class AppTestMemoryViewReversed(object): + spaceconfig = dict(usemodules=['array']) + def test_reversed_non_bytes(self): + import array + items = [1,2,3,9,7,5] + formats = ['h'] + for fmt in formats: + bytes = array.array(fmt, items) + view = memoryview(bytes) + bview = view.cast('b') + rview = bview.cast(fmt, shape=(2,3)) + raises(NotImplementedError, list, reversed(rview)) + assert rview.tolist() == [[1,2,3],[9,7,5]] + assert rview[::-1].tolist() == [[9,7,5], [1,2,3]] diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py --- a/pypy/tool/build_cffi_imports.py +++ b/pypy/tool/build_cffi_imports.py @@ -18,6 +18,7 @@ "lzma": "_lzma_build.py", "_decimal": "_decimal_build.py", "ssl": "_ssl_build.py", + # hashlib does not need to be built! It uses API calls from ssl "xx": None, # for testing: 'None' should be completely ignored } _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit