Author: Łukasz Langa <luk...@langa.pl>
Branch: py3.6
Changeset: r95864:baff18bae4fb
Date: 2019-02-06 12:01 +0100
http://bitbucket.org/pypy/pypy/changeset/baff18bae4fb/
Log: hg merge py3.5
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -4,8 +4,10 @@
*~
.*.swp
.idea
+.mypy_cache
.project
.pydevproject
+.vscode
__pycache__
.venv
.cache
diff --git a/lib_pypy/_csv.py b/lib_pypy/_csv.py
deleted file mode 100644
--- a/lib_pypy/_csv.py
+++ /dev/null
@@ -1,573 +0,0 @@
-"""CSV parsing and writing.
-
-This module provides classes that assist in the reading and writing
-of Comma Separated Value (CSV) files, and implements the interface
-described by PEP 305. Although many CSV files are simple to parse,
-the format is not formally defined by a stable specification and
-is subtle enough that parsing lines of a CSV file with something
-like line.split(\",\") is bound to fail. The module supports three
-basic APIs: reading, writing, and registration of dialects.
-
-
-DIALECT REGISTRATION:
-
-Readers and writers support a dialect argument, which is a convenient
-handle on a group of settings. When the dialect argument is a string,
-it identifies one of the dialects previously registered with the module.
-If it is a class or instance, the attributes of the argument are used as
-the settings for the reader or writer:
-
- class excel:
- delimiter = ','
- quotechar = '\"'
- escapechar = None
- doublequote = True
- skipinitialspace = False
- lineterminator = '\\r\\n'
- quoting = QUOTE_MINIMAL
-
-SETTINGS:
-
- * quotechar - specifies a one-character string to use as the
- quoting character. It defaults to '\"'.
- * delimiter - specifies a one-character string to use as the
- field separator. It defaults to ','.
- * skipinitialspace - specifies how to interpret whitespace which
- immediately follows a delimiter. It defaults to False, which
- means that whitespace immediately following a delimiter is part
- of the following field.
- * lineterminator - specifies the character sequence which should
- terminate rows.
- * quoting - controls when quotes should be generated by the writer.
- It can take on any of the following module constants:
-
- csv.QUOTE_MINIMAL means only when required, for example, when a
- field contains either the quotechar or the delimiter
- csv.QUOTE_ALL means that quotes are always placed around fields.
- csv.QUOTE_NONNUMERIC means that quotes are always placed around
- fields which do not parse as integers or floating point
- numbers.
- csv.QUOTE_NONE means that quotes are never placed around fields.
- * escapechar - specifies a one-character string used to escape
- the delimiter when quoting is set to QUOTE_NONE.
- * doublequote - controls the handling of quotes inside fields. When
- True, two consecutive quotes are interpreted as one during read,
- and when writing, each quote character embedded in the data is
- written as two quotes.
-"""
-
-__version__ = "1.0"
-
-QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE = range(4)
-_dialects = {}
-_field_limit = 128 * 1024 # max parsed field size
-
-class Error(Exception):
- pass
-
-class Dialect(object):
- """CSV dialect
-
- The Dialect type records CSV parsing and generation options."""
-
- __slots__ = ["_delimiter", "_doublequote", "_escapechar",
- "_lineterminator", "_quotechar", "_quoting",
- "_skipinitialspace", "_strict"]
-
- def __new__(cls, dialect, **kwargs):
-
- for name in kwargs:
- if '_' + name not in Dialect.__slots__:
- raise TypeError("unexpected keyword argument '%s'" %
- (name,))
-
- if dialect is not None:
- if isinstance(dialect, str):
- dialect = get_dialect(dialect)
-
- # Can we reuse this instance?
- if (isinstance(dialect, Dialect)
- and all(value is None for value in kwargs.values())):
- return dialect
-
- self = object.__new__(cls)
-
-
- def set_char(x):
- if x is None:
- return None
- if isinstance(x, str) and len(x) <= 1:
- return x
- raise TypeError("%r must be a 1-character string" % (name,))
- def set_str(x):
- if isinstance(x, str):
- return x
- raise TypeError("%r must be a string" % (name,))
- def set_quoting(x):
- if x in range(4):
- return x
- raise TypeError("bad 'quoting' value")
-
- attributes = {"delimiter": (',', set_char),
- "doublequote": (True, bool),
- "escapechar": (None, set_char),
- "lineterminator": ("\r\n", set_str),
- "quotechar": ('"', set_char),
- "quoting": (QUOTE_MINIMAL, set_quoting),
- "skipinitialspace": (False, bool),
- "strict": (False, bool),
- }
-
- # Copy attributes
- notset = object()
- for name in Dialect.__slots__:
- name = name[1:]
- value = notset
- if name in kwargs:
- value = kwargs[name]
- elif dialect is not None:
- value = getattr(dialect, name, notset)
-
- # mapping by name: (default, converter)
- if value is notset:
- value = attributes[name][0]
- if name == 'quoting' and not self.quotechar:
- value = QUOTE_NONE
- else:
- converter = attributes[name][1]
- if converter:
- value = converter(value)
-
- setattr(self, '_' + name, value)
-
- if not self.delimiter:
- raise TypeError("delimiter must be set")
-
- if self.quoting != QUOTE_NONE and not self.quotechar:
- raise TypeError("quotechar must be set if quoting enabled")
-
- if not self.lineterminator:
- raise TypeError("lineterminator must be set")
-
- return self
-
- delimiter = property(lambda self: self._delimiter)
- doublequote = property(lambda self: self._doublequote)
- escapechar = property(lambda self: self._escapechar)
- lineterminator = property(lambda self: self._lineterminator)
- quotechar = property(lambda self: self._quotechar)
- quoting = property(lambda self: self._quoting)
- skipinitialspace = property(lambda self: self._skipinitialspace)
- strict = property(lambda self: self._strict)
-
-
-def _call_dialect(dialect_inst, kwargs):
- return Dialect(dialect_inst, **kwargs)
-
-def register_dialect(name, dialect=None, **kwargs):
- """Create a mapping from a string name to a dialect class.
- dialect = csv.register_dialect(name, dialect)"""
- if not isinstance(name, str):
- raise TypeError("dialect name must be a string or unicode")
-
- dialect = _call_dialect(dialect, kwargs)
- _dialects[name] = dialect
-
-def unregister_dialect(name):
- """Delete the name/dialect mapping associated with a string name.\n
- csv.unregister_dialect(name)"""
- try:
- del _dialects[name]
- except KeyError:
- raise Error("unknown dialect")
-
-def get_dialect(name):
- """Return the dialect instance associated with name.
- dialect = csv.get_dialect(name)"""
- try:
- return _dialects[name]
- except KeyError:
- raise Error("unknown dialect")
-
-def list_dialects():
- """Return a list of all know dialect names
- names = csv.list_dialects()"""
- return list(_dialects)
-
-class Reader(object):
- """CSV reader
-
- Reader objects are responsible for reading and parsing tabular data
- in CSV format."""
-
-
- (START_RECORD, START_FIELD, ESCAPED_CHAR, IN_FIELD,
- IN_QUOTED_FIELD, ESCAPE_IN_QUOTED_FIELD, QUOTE_IN_QUOTED_FIELD,
- EAT_CRNL) = range(8)
-
- def __init__(self, iterator, dialect=None, **kwargs):
- self.dialect = _call_dialect(dialect, kwargs)
- self.input_iter = iter(iterator)
- self.line_num = 0
-
- self._parse_reset()
-
- def _parse_reset(self):
- self.field = ''
- self.fields = []
- self.state = self.START_RECORD
- self.numeric_field = False
-
- def __iter__(self):
- return self
-
- def __next__(self):
- self._parse_reset()
- while True:
- try:
- line = next(self.input_iter)
- except StopIteration:
- # End of input OR exception
- if len(self.field) > 0:
- raise Error("newline inside string")
- raise
-
- self.line_num += 1
-
- if '\0' in line:
- raise Error("line contains NULL byte")
- pos = 0
- while pos < len(line):
- pos = self._parse_process_char(line, pos)
- self._parse_eol()
-
- if self.state == self.START_RECORD:
- break
-
- fields = self.fields
- self.fields = []
- return fields
-
- def _parse_process_char(self, line, pos):
- c = line[pos]
- if self.state == self.IN_FIELD:
- # in unquoted field
- pos2 = pos
- while True:
- if c in '\n\r':
- # end of line - return [fields]
- if pos2 > pos:
- self._parse_add_char(line[pos:pos2])
- pos = pos2
- self._parse_save_field()
- self.state = self.EAT_CRNL
- elif c == self.dialect.escapechar:
- # possible escaped character
- pos2 -= 1
- self.state = self.ESCAPED_CHAR
- elif c == self.dialect.delimiter:
- # save field - wait for new field
- if pos2 > pos:
- self._parse_add_char(line[pos:pos2])
- pos = pos2
- self._parse_save_field()
- self.state = self.START_FIELD
- else:
- # normal character - save in field
- pos2 += 1
- if pos2 < len(line):
- c = line[pos2]
- continue
- break
- if pos2 > pos:
- self._parse_add_char(line[pos:pos2])
- pos = pos2 - 1
-
- elif self.state == self.START_RECORD:
- if c in '\n\r':
- self.state = self.EAT_CRNL
- else:
- self.state = self.START_FIELD
- # restart process
- self._parse_process_char(line, pos)
-
- elif self.state == self.START_FIELD:
- if c in '\n\r':
- # save empty field - return [fields]
- self._parse_save_field()
- self.state = self.EAT_CRNL
- elif (c == self.dialect.quotechar
- and self.dialect.quoting != QUOTE_NONE):
- # start quoted field
- self.state = self.IN_QUOTED_FIELD
- elif c == self.dialect.escapechar:
- # possible escaped character
- self.state = self.ESCAPED_CHAR
- elif c == ' ' and self.dialect.skipinitialspace:
- # ignore space at start of field
- pass
- elif c == self.dialect.delimiter:
- # save empty field
- self._parse_save_field()
- else:
- # begin new unquoted field
- if self.dialect.quoting == QUOTE_NONNUMERIC:
- self.numeric_field = True
- self._parse_add_char(c)
- self.state = self.IN_FIELD
-
- elif self.state == self.ESCAPED_CHAR:
- self._parse_add_char(c)
- self.state = self.IN_FIELD
-
- elif self.state == self.IN_QUOTED_FIELD:
- if c == self.dialect.escapechar:
- # possible escape character
- self.state = self.ESCAPE_IN_QUOTED_FIELD
- elif (c == self.dialect.quotechar
- and self.dialect.quoting != QUOTE_NONE):
- if self.dialect.doublequote:
- # doublequote; " represented by ""
- self.state = self.QUOTE_IN_QUOTED_FIELD
- else:
- #end of quote part of field
- self.state = self.IN_FIELD
- else:
- # normal character - save in field
- self._parse_add_char(c)
-
- elif self.state == self.ESCAPE_IN_QUOTED_FIELD:
- self._parse_add_char(c)
- self.state = self.IN_QUOTED_FIELD
-
- elif self.state == self.QUOTE_IN_QUOTED_FIELD:
- # doublequote - seen a quote in a quoted field
- if (c == self.dialect.quotechar
- and self.dialect.quoting != QUOTE_NONE):
- # save "" as "
- self._parse_add_char(c)
- self.state = self.IN_QUOTED_FIELD
- elif c == self.dialect.delimiter:
- # save field - wait for new field
- self._parse_save_field()
- self.state = self.START_FIELD
- elif c in '\r\n':
- # end of line - return [fields]
- self._parse_save_field()
- self.state = self.EAT_CRNL
- elif not self.dialect.strict:
- self._parse_add_char(c)
- self.state = self.IN_FIELD
- else:
- raise Error("'%c' expected after '%c'" %
- (self.dialect.delimiter, self.dialect.quotechar))
-
- elif self.state == self.EAT_CRNL:
- if c not in '\r\n':
- raise Error("new-line character seen in unquoted field - "
- "do you need to open the file "
- "in universal-newline mode?")
-
- else:
- raise RuntimeError("unknown state: %r" % (self.state,))
-
- return pos + 1
-
- def _parse_eol(self):
- if self.state == self.EAT_CRNL:
- self.state = self.START_RECORD
- elif self.state == self.START_RECORD:
- # empty line - return []
- pass
- elif self.state == self.IN_FIELD:
- # in unquoted field
- # end of line - return [fields]
- self._parse_save_field()
- self.state = self.START_RECORD
- elif self.state == self.START_FIELD:
- # save empty field - return [fields]
- self._parse_save_field()
- self.state = self.START_RECORD
- elif self.state == self.ESCAPED_CHAR:
- self._parse_add_char('\n')
- self.state = self.IN_FIELD
- elif self.state == self.IN_QUOTED_FIELD:
- pass
- elif self.state == self.ESCAPE_IN_QUOTED_FIELD:
- self._parse_add_char('\n')
- self.state = self.IN_QUOTED_FIELD
- elif self.state == self.QUOTE_IN_QUOTED_FIELD:
- # end of line - return [fields]
- self._parse_save_field()
- self.state = self.START_RECORD
- else:
- raise RuntimeError("unknown state: %r" % (self.state,))
-
- def _parse_save_field(self):
- field, self.field = self.field, ''
- if self.numeric_field:
- self.numeric_field = False
- field = float(field)
- self.fields.append(field)
-
- def _parse_add_char(self, c):
- if len(self.field) + len(c) > _field_limit:
- raise Error("field larger than field limit (%d)" % (_field_limit))
- self.field += c
-
-
-class Writer(object):
- """CSV writer
-
- Writer objects are responsible for generating tabular data
- in CSV format from sequence input."""
-
- def __init__(self, file, dialect=None, **kwargs):
- if not (hasattr(file, 'write') and callable(file.write)):
- raise TypeError("argument 1 must have a 'write' method")
- self.writeline = file.write
- self.dialect = _call_dialect(dialect, kwargs)
-
- def _join_reset(self):
- self.rec = []
- self.num_fields = 0
-
- def _join_append(self, field, quoted, quote_empty):
- dialect = self.dialect
- # If this is not the first field we need a field separator
- if self.num_fields > 0:
- self.rec.append(dialect.delimiter)
-
- if dialect.quoting == QUOTE_NONE:
- need_escape = tuple(dialect.lineterminator) + (
- dialect.escapechar, # escapechar always first
- dialect.delimiter, dialect.quotechar)
-
- else:
- for c in tuple(dialect.lineterminator) + (
- dialect.delimiter, dialect.escapechar):
- if c and c in field:
- quoted = True
-
- need_escape = ()
- if dialect.quotechar in field:
- if dialect.doublequote:
- field = field.replace(dialect.quotechar,
- dialect.quotechar * 2)
- quoted = True
- else:
- need_escape = (dialect.quotechar,)
-
-
- for c in need_escape:
- if c and c in field:
- if not dialect.escapechar:
- raise Error("need to escape, but no escapechar set")
- field = field.replace(c, dialect.escapechar + c)
-
- # If field is empty check if it needs to be quoted
- if field == '' and quote_empty:
- if dialect.quoting == QUOTE_NONE:
- raise Error("single empty field record must be quoted")
- quoted = 1
-
- if quoted:
- field = dialect.quotechar + field + dialect.quotechar
-
- self.rec.append(field)
- self.num_fields += 1
-
-
-
- def writerow(self, row):
- dialect = self.dialect
- try:
- rowlen = len(row)
- except TypeError:
- raise Error("sequence expected")
-
- # join all fields in internal buffer
- self._join_reset()
-
- for field in row:
- quoted = False
- if dialect.quoting == QUOTE_NONNUMERIC:
- try:
- float(field)
- except:
- quoted = True
- # This changed since 2.5:
- # quoted = not isinstance(field, (int, long, float))
- elif dialect.quoting == QUOTE_ALL:
- quoted = True
-
- if field is None:
- value = ""
- elif isinstance(field, float):
- value = repr(field)
- else:
- value = str(field)
- self._join_append(value, quoted, rowlen == 1)
-
- # add line terminator
- self.rec.append(dialect.lineterminator)
-
- self.writeline(''.join(self.rec))
-
- def writerows(self, rows):
- for row in rows:
- self.writerow(row)
-
-def reader(*args, **kwargs):
- """
- csv_reader = reader(iterable [, dialect='excel']
- [optional keyword args])
- for row in csv_reader:
- process(row)
-
- The "iterable" argument can be any object that returns a line
- of input for each iteration, such as a file object or a list. The
- optional \"dialect\" parameter is discussed below. The function
- also accepts optional keyword arguments which override settings
- provided by the dialect.
-
- The returned object is an iterator. Each iteration returns a row
- of the CSV file (which can span multiple input lines)"""
-
- return Reader(*args, **kwargs)
-
-def writer(*args, **kwargs):
- """
- csv_writer = csv.writer(fileobj [, dialect='excel']
- [optional keyword args])
- for row in sequence:
- csv_writer.writerow(row)
-
- [or]
-
- csv_writer = csv.writer(fileobj [, dialect='excel']
- [optional keyword args])
- csv_writer.writerows(rows)
-
- The \"fileobj\" argument can be any object that supports the file API."""
- return Writer(*args, **kwargs)
-
-
-undefined = object()
-def field_size_limit(limit=undefined):
- """Sets an upper limit on parsed fields.
- csv.field_size_limit([limit])
-
- Returns old limit. If limit is not given, no new limit is set and
- the old limit is returned"""
-
- global _field_limit
- old_limit = _field_limit
-
- if limit is not undefined:
- if not isinstance(limit, int):
- raise TypeError("int expected, got %s" %
- (limit.__class__.__name__,))
- _field_limit = limit
-
- return old_limit
diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py
deleted file mode 100644
--- a/lib_pypy/pwd.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# indirectly based on ctypes implementation: Victor Stinner, 2008-05-08
-"""
-This module provides access to the Unix password database.
-It is available on all Unix versions.
-
-Password database entries are reported as 7-tuples containing the following
-items from the password database (see `<pwd.h>'), in order:
-pw_name, pw_passwd, pw_uid, pw_gid, pw_gecos, pw_dir, pw_shell.
-The uid and gid items are integers, all others are strings. An
-exception is raised if the entry asked for cannot be found.
-"""
-
-from _pwdgrp_cffi import ffi, lib
-import _structseq
-import _thread
-_lock = _thread.allocate_lock()
-
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
-
-
-class struct_passwd(metaclass=_structseq.structseqtype):
- """
- pwd.struct_passwd: Results from getpw*() routines.
-
- This object may be accessed either as a tuple of
- (pw_name,pw_passwd,pw_uid,pw_gid,pw_gecos,pw_dir,pw_shell)
- or via the object attributes as named in the above tuple.
- """
- name = "pwd.struct_passwd"
-
- pw_name = _structseq.structseqfield(0)
- pw_passwd = _structseq.structseqfield(1)
- pw_uid = _structseq.structseqfield(2)
- pw_gid = _structseq.structseqfield(3)
- pw_gecos = _structseq.structseqfield(4)
- pw_dir = _structseq.structseqfield(5)
- pw_shell = _structseq.structseqfield(6)
-
-
-def _mkpwent(pw):
- return struct_passwd([
- ffi.string(pw.pw_name),
- ffi.string(pw.pw_passwd),
- pw.pw_uid,
- pw.pw_gid,
- ffi.string(pw.pw_gecos),
- ffi.string(pw.pw_dir),
- ffi.string(pw.pw_shell)])
-
-@builtinify
-def getpwuid(uid):
- """
- getpwuid(uid) -> (pw_name,pw_passwd,pw_uid,
- pw_gid,pw_gecos,pw_dir,pw_shell)
- Return the password database entry for the given numeric user ID.
- See pwd.__doc__ for more on password database entries.
- """
- with _lock:
- pw = lib.getpwuid(uid)
- if not pw:
- raise KeyError("getpwuid(): uid not found: %s" % uid)
- return _mkpwent(pw)
-
-@builtinify
-def getpwnam(name):
- """
- getpwnam(name) -> (pw_name,pw_passwd,pw_uid,
- pw_gid,pw_gecos,pw_dir,pw_shell)
- Return the password database entry for the given user name.
- See pwd.__doc__ for more on password database entries.
- """
- if not isinstance(name, basestring):
- raise TypeError("expected string")
- name = str(name)
- with _lock:
- pw = lib.getpwnam(name)
- if not pw:
- raise KeyError("getpwname(): name not found: %s" % name)
- return _mkpwent(pw)
-
-@builtinify
-def getpwall():
- """
- getpwall() -> list_of_entries
- Return a list of all available password database entries, in arbitrary
order.
- See pwd.__doc__ for more on password database entries.
- """
- users = []
- with _lock:
- lib.setpwent()
- while True:
- pw = lib.getpwent()
- if not pw:
- break
- users.append(_mkpwent(pw))
- lib.endpwent()
- return users
-
-__all__ = ('struct_passwd', 'getpwuid', 'getpwnam', 'getpwall')
-
-if __name__ == "__main__":
-# Uncomment next line to test CPython implementation
-# from pwd import getpwuid, getpwnam, getpwall
- from os import getuid
- uid = getuid()
- pw = getpwuid(uid)
- print("uid %s: %s" % (pw.pw_uid, pw))
- name = pw.pw_name
- print("name %r: %s" % (name, getpwnam(name)))
- print("All:")
- for pw in getpwall():
- print(pw)
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -113,20 +113,9 @@
Wrapper around zlib's z_stream structure which provides convenient
compression functionality.
"""
- def __init__(self, space, level=rzlib.Z_DEFAULT_COMPRESSION,
- method=rzlib.Z_DEFLATED, # \
- wbits=rzlib.MAX_WBITS, # \ undocumented
- memLevel=rzlib.DEF_MEM_LEVEL, # / parameters
- strategy=rzlib.Z_DEFAULT_STRATEGY, # /
- zdict=None):
+ def __init__(self, space, stream):
ZLibObject.__init__(self, space)
- try:
- self.stream = rzlib.deflateInit(level, method, wbits,
- memLevel, strategy, zdict=zdict)
- except rzlib.RZlibError as e:
- raise zlib_error(space, e.msg)
- except ValueError:
- raise oefmt(space.w_ValueError, "Invalid initialization option")
+ self.stream = stream
self.register_finalizer(space)
def _finalize_(self):
@@ -158,6 +147,20 @@
raise zlib_error(space, e.msg)
return space.newbytes(result)
+ def copy(self, space):
+ """
+ copy() -- Return a copy of the compression object.
+ """
+ try:
+ self.lock()
+ try:
+ copied = rzlib.deflateCopy(self.stream)
+ finally:
+ self.unlock()
+ except rzlib.RZlibError as e:
+ raise zlib_error(space, e.msg)
+ return Compress(space=space, stream=copied)
+
@unwrap_spec(mode="c_int")
def flush(self, space, mode=rzlib.Z_FINISH):
"""
@@ -203,16 +206,23 @@
zdict = None
else:
zdict = space.charbuf_w(w_zdict)
- stream = space.allocate_instance(Compress, w_subtype)
- stream = space.interp_w(Compress, stream)
- Compress.__init__(stream, space, level,
- method, wbits, memLevel, strategy, zdict)
- return stream
+ w_stream = space.allocate_instance(Compress, w_subtype)
+ w_stream = space.interp_w(Compress, w_stream)
+ try:
+ stream = rzlib.deflateInit(level, method, wbits, memLevel, strategy,
+ zdict=zdict)
+ except rzlib.RZlibError as e:
+ raise zlib_error(space, e.msg)
+ except ValueError:
+ raise oefmt(space.w_ValueError, "Invalid initialization option")
+ Compress.__init__(w_stream, space, stream)
+ return w_stream
Compress.typedef = TypeDef(
'Compress',
__new__ = interp2app(Compress___new__),
+ copy = interp2app(Compress.copy),
compress = interp2app(Compress.compress),
flush = interp2app(Compress.flush),
__doc__ = """compressobj([level]) -- Return a compressor object.
@@ -226,7 +236,7 @@
Wrapper around zlib's z_stream structure which provides convenient
decompression functionality.
"""
- def __init__(self, space, wbits=rzlib.MAX_WBITS, zdict=None):
+ def __init__(self, space, stream, zdict, unused_data, unconsumed_tail):
"""
Initialize a new decompression object.
@@ -236,16 +246,12 @@
inflateInit2.
"""
ZLibObject.__init__(self, space)
- self.unused_data = ''
- self.unconsumed_tail = ''
+
+ self.stream = stream
+ self.zdict = zdict
+ self.unused_data = unused_data
+ self.unconsumed_tail = unconsumed_tail
self.eof = False
- try:
- self.stream = rzlib.inflateInit(wbits, zdict=zdict)
- except rzlib.RZlibError as e:
- raise zlib_error(space, e.msg)
- except ValueError:
- raise oefmt(space.w_ValueError, "Invalid initialization option")
- self.zdict = zdict
self.register_finalizer(space)
def _finalize_(self):
@@ -295,6 +301,30 @@
self._save_unconsumed_input(data, finished, unused_len)
return space.newbytes(string)
+ def copy(self, space):
+ """
+ copy() -- Return a copy of the decompression object.
+ """
+ try:
+ self.lock()
+ try:
+ if not self.stream:
+ raise zlib_error(space,
+ "decompressor object already flushed")
+ copied = rzlib.inflateCopy(self.stream)
+ finally:
+ self.unlock()
+ except rzlib.RZlibError as e:
+ raise zlib_error(space, e.msg)
+
+ return Decompress(
+ space=space,
+ stream=copied,
+ unused_data=self.unused_data,
+ unconsumed_tail=self.unconsumed_tail,
+ zdict=self.zdict,
+ )
+
def flush(self, space, w_length=None):
"""
flush( [length] ) -- This is kept for backward compatibility,
@@ -331,10 +361,16 @@
zdict = None
else:
zdict = space.charbuf_w(w_zdict)
- stream = space.allocate_instance(Decompress, w_subtype)
- stream = space.interp_w(Decompress, stream)
- Decompress.__init__(stream, space, wbits, zdict)
- return stream
+ w_stream = space.allocate_instance(Decompress, w_subtype)
+ w_stream = space.interp_w(Decompress, w_stream)
+ try:
+ stream = rzlib.inflateInit(wbits, zdict=zdict)
+ except rzlib.RZlibError as e:
+ raise zlib_error(space, e.msg)
+ except ValueError:
+ raise oefmt(space.w_ValueError, "Invalid initialization option")
+ Decompress.__init__(w_stream, space, stream, zdict, '', '')
+ return w_stream
def default_buffer_size(space):
return space.newint(rzlib.OUTPUT_BUFFER_SIZE)
@@ -342,6 +378,7 @@
Decompress.typedef = TypeDef(
'Decompress',
__new__ = interp2app(Decompress___new__),
+ copy = interp2app(Decompress.copy),
decompress = interp2app(Decompress.decompress),
flush = interp2app(Decompress.flush),
unused_data = interp_attrproperty('unused_data', Decompress,
wrapfn="newbytes"),
diff --git a/pypy/module/zlib/test/test_zlib.py
b/pypy/module/zlib/test/test_zlib.py
--- a/pypy/module/zlib/test/test_zlib.py
+++ b/pypy/module/zlib/test/test_zlib.py
@@ -10,6 +10,13 @@
except ImportError:
py.test.skip("no zlib module on this host Python")
+from pypy.interpreter.gateway import interp2app
+try:
+ from pypy.module.zlib import interp_zlib
+ from rpython.rlib import rzlib
+except ImportError:
+ import py; py.test.skip("no zlib C library on this machine")
+
class AppTestZlib(object):
spaceconfig = dict(usemodules=['zlib'])
@@ -29,6 +36,22 @@
py.path.local(pypy.__file__).dirpath().dirpath()
.join('LICENSE').read())
+ def intentionally_break_a_z_stream(space, w_zobj):
+ """
+ Intentionally break the z_stream associated with a
+ compressobj or decompressobj in a way that causes their copy
+ methods to raise RZlibErrors.
+ """
+ from rpython.rtyper.lltypesystem import rffi, lltype
+ w_zobj.stream.c_zalloc = rffi.cast(
+ lltype.typeOf(w_zobj.stream.c_zalloc),
+ rzlib.Z_NULL,
+ )
+
+ cls.w_intentionally_break_a_z_stream = cls.space.wrap(
+ interp2app(intentionally_break_a_z_stream),
+ )
+
def test_def_buf_size(self):
assert self.zlib.DEF_BUF_SIZE >= 0
@@ -330,3 +353,54 @@
dco = zlib.decompressobj(wbits=-zlib.MAX_WBITS, zdict=zdict)
uncomp = dco.decompress(comp) + dco.flush()
assert zdict == uncomp
+
+ def test_decompress_copy(self):
+ decompressor = self.zlib.decompressobj()
+ d1 = decompressor.decompress(self.compressed[:10])
+ assert d1
+
+ copied = decompressor.copy()
+
+ from_copy = copied.decompress(self.compressed[10:])
+ from_decompressor = decompressor.decompress(self.compressed[10:])
+
+ assert (d1 + from_copy) == (d1 + from_decompressor)
+
+ def test_unsuccessful_decompress_copy(self):
+ decompressor = self.zlib.decompressobj()
+ self.intentionally_break_a_z_stream(zobj=decompressor)
+ raises(self.zlib.error, decompressor.copy)
+
+ def test_decompress_copy_carries_along_state(self):
+ """
+ Decompressor.unused_data and unconsumed_tail are carried along when a
+ copy is done.
+ """
+ decompressor = self.zlib.decompressobj()
+ decompressor.decompress(self.compressed, max_length=5)
+ unconsumed_tail = decompressor.unconsumed_tail
+ assert unconsumed_tail
+ assert decompressor.copy().unconsumed_tail == unconsumed_tail
+
+ decompressor.decompress(decompressor.unconsumed_tail)
+ decompressor.decompress(b"xxx")
+ unused_data = decompressor.unused_data
+ assert unused_data
+ assert decompressor.copy().unused_data == unused_data
+
+ def test_compress_copy(self):
+ compressor = self.zlib.compressobj()
+ d1 = compressor.compress(self.expanded[:10])
+ assert d1
+
+ copied = compressor.copy()
+
+ from_copy = copied.compress(self.expanded[10:])
+ from_compressor = compressor.compress(self.expanded[10:])
+
+ assert (d1 + from_copy) == (d1 + from_compressor)
+
+ def test_unsuccessful_compress_copy(self):
+ compressor = self.zlib.compressobj()
+ self.intentionally_break_a_z_stream(zobj=compressor)
+ raises(self.zlib.error, compressor.copy)
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -121,6 +121,8 @@
pypydir.ensure('include', dir=True)
if sys.platform == 'win32':
+ os.environ['PATH'] = str(basedir.join('externals').join('bin')) + ';'
+ \
+ os.environ.get('PATH', '')
src,tgt = binaries[0]
pypyw = src.new(purebasename=src.purebasename + 'w')
if pypyw.exists():
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -1406,6 +1406,11 @@
return _ResizableListSupportingRawPtr(lst)
def nonmoving_raw_ptr_for_resizable_list(lst):
+ if must_split_gc_address_space():
+ raise ValueError
+ return _nonmoving_raw_ptr_for_resizable_list(lst)
+
+def _nonmoving_raw_ptr_for_resizable_list(lst):
assert isinstance(lst, _ResizableListSupportingRawPtr)
return lst._nonmoving_raw_ptr_for_resizable_list()
@@ -1450,7 +1455,7 @@
return hop.inputarg(hop.args_r[0], 0)
class Entry(ExtRegistryEntry):
- _about_ = nonmoving_raw_ptr_for_resizable_list
+ _about_ = _nonmoving_raw_ptr_for_resizable_list
def compute_result_annotation(self, s_list):
from rpython.rtyper.lltypesystem import lltype, rffi
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -35,6 +35,7 @@
MAX_WBITS MAX_MEM_LEVEL
Z_BEST_SPEED Z_BEST_COMPRESSION Z_DEFAULT_COMPRESSION
Z_FILTERED Z_HUFFMAN_ONLY Z_DEFAULT_STRATEGY Z_NEED_DICT
+ Z_NULL
'''.split()
class SimpleCConfig:
@@ -141,6 +142,7 @@
rffi.INT)
_deflate = zlib_external('deflate', [z_stream_p, rffi.INT], rffi.INT)
+_deflateCopy = zlib_external('deflateCopy', [z_stream_p, z_stream_p], rffi.INT)
_deflateEnd = zlib_external('deflateEnd', [z_stream_p], rffi.INT,
releasegil=False)
@@ -160,6 +162,7 @@
rffi.INT)
_inflate = zlib_external('inflate', [z_stream_p, rffi.INT], rffi.INT)
+_inflateCopy = zlib_external('inflateCopy', [z_stream_p, z_stream_p], rffi.INT)
_inflateEnd = zlib_external('inflateEnd', [z_stream_p], rffi.INT,
releasegil=False)
@@ -290,6 +293,19 @@
lltype.free(stream, flavor='raw')
+def deflateCopy(source):
+ """
+ Allocate and return an independent copy of the provided stream object.
+ """
+ dest = deflateInit()
+ err = _deflateCopy(dest, source)
+ if err != Z_OK:
+ deflateEnd(dest)
+ raise RZlibError.fromstream(source, err,
+ "while copying compression object")
+ return dest
+
+
def deflateEnd(stream):
"""
Free the resources associated with the deflate stream.
@@ -330,6 +346,19 @@
lltype.free(stream, flavor='raw')
+def inflateCopy(source):
+ """
+ Allocate and return an independent copy of the provided stream object.
+ """
+ dest = inflateInit()
+ err = _inflateCopy(dest, source)
+ if err != Z_OK:
+ inflateEnd(dest)
+ raise RZlibError.fromstream(source, err,
+ "while copying decompression object")
+ return dest
+
+
def inflateEnd(stream):
"""
Free the resources associated with the inflate stream.
diff --git a/rpython/rlib/test/test_rzlib.py b/rpython/rlib/test/test_rzlib.py
--- a/rpython/rlib/test/test_rzlib.py
+++ b/rpython/rlib/test/test_rzlib.py
@@ -246,6 +246,108 @@
rzlib.deflateEnd(stream)
+def test_compress_copy():
+ """
+ inflateCopy produces an independent copy of a stream.
+ """
+
+ stream = rzlib.deflateInit()
+
+ bytes1 = rzlib.compress(stream, expanded[:10])
+ assert bytes1
+
+ copied = rzlib.deflateCopy(stream)
+
+ bytes_stream = rzlib.compress(
+ stream,
+ expanded[10:],
+ rzlib.Z_FINISH,
+ )
+ assert bytes1 + bytes_stream == compressed
+ rzlib.deflateEnd(stream)
+
+ bytes_copy = rzlib.compress(
+ copied,
+ expanded[10:],
+ rzlib.Z_FINISH,
+ )
+ rzlib.deflateEnd(copied)
+ assert bytes1 + bytes_copy == compressed
+
+
+def test_unsuccessful_compress_copy():
+ """
+ Errors during unsuccesful deflateCopy operations raise RZlibErrors.
+ """
+ stream = rzlib.deflateInit()
+
+ # From zlib.h:
+ #
+ # "deflateCopy returns [...] Z_STREAM_ERROR if the source stream
+ # state was inconsistent (such as zalloc being Z_NULL)"
+ from rpython.rtyper.lltypesystem import rffi, lltype
+ stream.c_zalloc = rffi.cast(lltype.typeOf(stream.c_zalloc), rzlib.Z_NULL)
+
+ exc = py.test.raises(rzlib.RZlibError, rzlib.deflateCopy, stream)
+ msg = "Error -2 while copying compression object: inconsistent stream
state"
+ assert str(exc.value) == msg
+ rzlib.deflateEnd(stream)
+
+
+def test_decompress_copy():
+ """
+ inflateCopy produces an independent copy of a stream.
+ """
+
+ stream = rzlib.inflateInit()
+
+ bytes1, finished1, unused1 = rzlib.decompress(stream, compressed[:10])
+ assert bytes1
+ assert finished1 is False
+
+ copied = rzlib.inflateCopy(stream)
+
+ bytes_stream, finished_stream, unused_stream = rzlib.decompress(
+ stream,
+ compressed[10:],
+ rzlib.Z_FINISH,
+ )
+ assert bytes1 + bytes_stream == expanded
+ assert finished_stream is True
+ assert unused1 == 0
+ assert unused_stream == 0
+ rzlib.inflateEnd(stream)
+
+ bytes_copy, finished_copy, unused_copy = rzlib.decompress(
+ copied,
+ compressed[10:],
+ rzlib.Z_FINISH,
+ )
+ rzlib.inflateEnd(copied)
+ assert bytes1 + bytes_copy == expanded
+ assert finished_copy is True
+ assert unused_copy == 0
+
+
+def test_unsuccessful_decompress_copy():
+ """
+ Errors during unsuccesful inflateCopy operations raise RZlibErrors.
+ """
+ stream = rzlib.inflateInit()
+
+ # From zlib.h:
+ #
+ # "inflateCopy returns [...] Z_STREAM_ERROR if the source stream
+ # state was inconsistent (such as zalloc being Z_NULL)"
+ from rpython.rtyper.lltypesystem import rffi, lltype
+ stream.c_zalloc = rffi.cast(lltype.typeOf(stream.c_zalloc), rzlib.Z_NULL)
+
+ exc = py.test.raises(rzlib.RZlibError, rzlib.inflateCopy, stream)
+ msg = "Error -2 while copying decompression object: inconsistent stream
state"
+ assert str(exc.value) == msg
+ rzlib.inflateEnd(stream)
+
+
def test_cornercases():
"""
Test degenerate arguments.
diff --git a/rpython/rtyper/lltypesystem/module/test/math_cases.py
b/rpython/rtyper/lltypesystem/module/test/math_cases.py
--- a/rpython/rtyper/lltypesystem/module/test/math_cases.py
+++ b/rpython/rtyper/lltypesystem/module/test/math_cases.py
@@ -59,9 +59,6 @@
('copysign', (1.5, -0.0), -1.5),
('copysign', (1.5, INFINITY), 1.5),
('copysign', (1.5, -INFINITY), -1.5),
- ]
- if sys.platform != 'win32': # all NaNs seem to be negative there...?
- IRREGCASES += [
('copysign', (1.5, NAN), 1.5),
('copysign', (1.75, -NAN), -1.75), # special case for -NAN here
]
diff --git a/rpython/translator/c/primitive.py
b/rpython/translator/c/primitive.py
--- a/rpython/translator/c/primitive.py
+++ b/rpython/translator/c/primitive.py
@@ -123,9 +123,9 @@
return '(-Py_HUGE_VAL)'
elif math.isnan(value):
if is_positive_nan(value):
- return '(Py_HUGE_VAL/Py_HUGE_VAL)'
+ return '(_PyPy_dg_stdnan(0))'
else:
- return '(-(Py_HUGE_VAL/Py_HUGE_VAL))'
+ return '(_PyPy_dg_stdnan(1))'
else:
x = repr(value)
assert not x.startswith('n')
@@ -142,9 +142,9 @@
elif math.isnan(value):
# XXX are these expressions ok?
if is_positive_nan(value):
- return '((float)(Py_HUGE_VAL/Py_HUGE_VAL))'
+ return '((float)(_PyPy_dg_stdnan(0)))'
else:
- return '(-(float)(Py_HUGE_VAL/Py_HUGE_VAL))'
+ return '((float)(_PyPy_dg_stdnan(1)))'
else:
return repr(value) + 'f'
diff --git a/rpython/translator/c/src/support.c
b/rpython/translator/c/src/support.c
--- a/rpython/translator/c/src/support.c
+++ b/rpython/translator/c/src/support.c
@@ -9,6 +9,26 @@
#include <stdlib.h>
/*** misc ***/
+#define Sign_bit 0x80000000
+#define NAN_WORD0 0x7ff80000
+#define NAN_WORD1 0
+#define PY_UINT32_T unsigned int
+
+#ifndef __BIG_ENDIAN__
+#define IEEE_8087
+#endif
+
+#ifdef IEEE_8087
+#define word0(x) (x)->L[1]
+#define word1(x) (x)->L[0]
+#else
+#define word0(x) (x)->L[0]
+#define word1(x) (x)->L[1]
+#endif
+#define dval(x) (x)->d
+
+typedef PY_UINT32_T ULong;
+typedef union { double d; ULong L[2]; } U;
RPY_EXTERN
void RPyAssertFailed(const char* filename, long lineno,
@@ -25,3 +45,20 @@
fprintf(stderr, "Invalid RPython operation (NULL ptr or bad array index)\n");
abort();
}
+
+/* Return a 'standard' NaN value.
+ There are exactly two quiet NaNs that don't arise by 'quieting' signaling
+ NaNs (see IEEE 754-2008, section 6.2.1). If sign == 0, return the one whose
+ sign bit is cleared. Otherwise, return the one whose sign bit is set.
+*/
+
+double
+_PyPy_dg_stdnan(int sign)
+{
+ U rv;
+ word0(&rv) = NAN_WORD0;
+ word1(&rv) = NAN_WORD1;
+ if (sign)
+ word0(&rv) |= Sign_bit;
+ return dval(&rv);
+}
diff --git a/rpython/translator/c/src/support.h
b/rpython/translator/c/src/support.h
--- a/rpython/translator/c/src/support.h
+++ b/rpython/translator/c/src/support.h
@@ -38,6 +38,9 @@
RPY_EXTERN
void RPyAbort(void);
+RPY_EXTERN
+double _PyPy_dg_stdnan(int sign);
+
#if defined(RPY_LL_ASSERT) || defined(RPY_SANDBOXED)
/* obscure macros that can be used as expressions and lvalues to refer
* to a field of a structure or an item in an array in a "safe" way --
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
