Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-ijson for openSUSE:Factory checked in at 2023-07-26 13:24:30 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-ijson (Old) and /work/SRC/openSUSE:Factory/.python-ijson.new.15225 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-ijson" Wed Jul 26 13:24:30 2023 rev:5 rq:1100746 version:3.2.3 Changes: -------- --- /work/SRC/openSUSE:Factory/python-ijson/python-ijson.changes 2023-06-11 19:58:51.456454069 +0200 +++ /work/SRC/openSUSE:Factory/.python-ijson.new.15225/python-ijson.changes 2023-07-26 13:25:36.844604415 +0200 @@ -1,0 +2,25 @@ +Wed Jul 26 06:47:55 UTC 2023 - Dirk Müller <[email protected]> + +- update to 3.2.3: + * Fixed several issues in the ``yajl2_c`` backend + and its async generators that were only made apparent + when running it with PyPy and/or a CPython debug build (#101). + * Adapted ``yajl2_c`` async generators to work against PyPy + shortcomings + * Fixed compilation and ``async`` support + of the ``yajl2_c`` backend in pyhthon 3.12 (#98). + * Check ``IJSON_BUILD_YAJL2C`` environment variable + when building ijson to force/skip building the + ``yajl2_c`` backend (#102). + * Added support for Python 3.12. + * Fixed a memory leak in the ``yajl2_c`` backend + triggered only when the underlying ``yajl`` functions + reported a failure (#97). + * New ``ijson.dump`` command-line utility + * Fixed bug in ``yajl2_c`` backend introduced in 3.1.2 + where random crashes could occur due to an unsafe reference decrement + when constructing the parse/items/kvitems generators (#66). + * Mark Python 3.10 and 3.11 as explicitly supported. +- drop tests_asyncio.py source - this comes in the tarball now + +------------------------------------------------------------------- Old: ---- ijson-3.1.4.tar.gz tests_asyncio.py New: ---- ijson-3.2.3.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-ijson.spec ++++++ --- /var/tmp/diff_new_pack.xxkvuf/_old 2023-07-26 13:25:37.336607383 +0200 +++ /var/tmp/diff_new_pack.xxkvuf/_new 2023-07-26 13:25:37.340607407 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-ijson # -# Copyright (c) 2021 SUSE LLC +# Copyright (c) 2023 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -18,15 +18,13 @@ %{?sle15_python_module_pythons} Name: python-ijson -Version: 3.1.4 +Version: 3.2.3 Release: 0 Summary: Iterative JSON parser with a standard Python iterator interface License: BSD-3-Clause Group: Development/Languages/Python URL: https://github.com/ICRAR/ijson Source: https://files.pythonhosted.org/packages/source/i/ijson/ijson-%{version}.tar.gz -# https://github.com/ICRAR/ijson/pull/26 -Source1: https://raw.githubusercontent.com/ICRAR/ijson/master/tests_asyncio.py BuildRequires: %{python_module devel} BuildRequires: %{python_module setuptools} BuildRequires: fdupes @@ -40,7 +38,6 @@ %prep %setup -q -n ijson-%{version} -cp %{SOURCE1} . %build export CFLAGS="%{optflags}" ++++++ ijson-3.1.4.tar.gz -> ijson-3.2.3.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/CHANGELOG.md new/ijson-3.2.3/CHANGELOG.md --- old/ijson-3.1.4/CHANGELOG.md 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/CHANGELOG.md 2023-07-22 07:17:10.000000000 +0200 @@ -1,5 +1,56 @@ # Changelog +## [3.2.3] + +* Fixed several issues in the ``yajl2_c`` backend + and its async generators + that were only made apparent + when running it with PyPy + and/or a CPython debug build (#101). + As part of that, + an issue was found and fixed in PyPy itself + affecting all versions up to 7.3.12, + so users will need to wait until the next version is released + to be able to use async generators + (https://foss.heptapod.net/pypy/pypy/-/issues/3956). +* Adapted ``yajl2_c`` async generators + to work against PyPy shortcomings + (https://foss.heptapod.net/pypy/pypy/-/issues/3965). + +## [3.2.2] + +* Fixed compilation and ``async`` support + of the ``yajl2_c`` backend in pyhthon 3.12 (#98). +* Check ``IJSON_BUILD_YAJL2C`` environment variable + when building ijson + to force/skip building the ``yajl2_c`` backend (#102). + +## [3.2.1] + +* Added support for Python 3.12. +* Fixed a memory leak in the ``yajl2_c`` backend + triggered only when the underlying ``yajl`` functions + reported a failure (#97). + +## [3.2.0.post0] + +* Fixed minor README rendering issues + that prevented upload of 3.2.0 distributions + to PyPI. + +## [3.2.0] + +* New ``ijson.dump`` command-line utility + for simple inspection of the ijson iteration process. + This tool should be useful for new users + who are usually confused with how to use the library, + and the prefix in particular. +* Fixed bug in ``yajl2_c`` backend introduced in 3.1.2 + where random crashes could occur + due to an unsafe reference decrement + when constructing the parse/items/kvitems generators (#66). +* Mark Python 3.10 and 3.11 as explicitly supported. + ## [3.1.4] * Fixed bug in ``yajl2_c`` backend introduced in 3.1.0 @@ -35,7 +86,7 @@ * Moved binary wheel generation from GitHub Actions to Travis. This gained us binary ARM wheels, - wihch are becoming increasingly popular (#35) + which are becoming increasingly popular (#35) ## [3.1.2] @@ -79,13 +130,13 @@ of the ``yajl2_c`` backend (by internally avoiding unnecessary string concatenations). Local tests show a performance improvement of up to ~15%, - but milage might vary depending on your use case and system. + but mileage might vary depending on your use case and system. * The "raw" functions ``basic_parse``, ``parse``, ``items`` and ``kvitems`` can now be used with different types of inputs. In particular they accept not only file-like objects, but also asynchronous file-like objects, behaving like their ``*_async`` counterparts. - They also accept ``bytes`` and ``str`` objects direclty + They also accept ``bytes`` and ``str`` objects directly (and ``unicode`` objects in python 2.7). Finally, they also accept iterables, in which case they behave like the ``ijson.common.*`` functions, @@ -95,7 +146,7 @@ Users should use the ``ijson.*`` routines instead, which now accept event iterables. * New ``ijson.get_backend`` function - for users to import a backend programatically + for users to import a backend programmatically (without having to manually use importlib). * New ``IJSON_BACKEND`` environment variable can be used to choose the default backend to be exposed by ijson. @@ -138,7 +189,7 @@ under the ``ijson.common`` module (#27). These functions take an events iterable instead of a file and are backend-independent (which is not great for performance). - They were accidentaly removed in the redesign of ijson 3.0, + They were accidentally removed in the redesign of ijson 3.0, which is why they are coming back. In the future they will slowly transition into being backend-specific rather than independent. @@ -279,3 +330,8 @@ [3.1.2.post0]: https://github.com/ICRAR/ijson/releases/tag/v3.1.2.post0 [3.1.3]: https://github.com/ICRAR/ijson/releases/tag/v3.1.3 [3.1.4]: https://github.com/ICRAR/ijson/releases/tag/v3.1.4 +[3.2.0]: https://github.com/ICRAR/ijson/releases/tag/v3.2.0 +[3.2.0.post0]: https://github.com/ICRAR/ijson/releases/tag/v3.2.0.post0 +[3.2.1]: https://github.com/ICRAR/ijson/releases/tag/v3.2.1 +[3.2.2]: https://github.com/ICRAR/ijson/releases/tag/v3.2.2 +[3.2.3]: https://github.com/ICRAR/ijson/releases/tag/v3.2.3 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/PKG-INFO new/ijson-3.2.3/PKG-INFO --- old/ijson-3.1.4/PKG-INFO 2021-03-02 11:05:01.667155300 +0100 +++ new/ijson-3.2.3/PKG-INFO 2023-07-22 07:17:12.000000000 +0200 @@ -1,13 +1,13 @@ Metadata-Version: 2.1 Name: ijson -Version: 3.1.4 +Version: 3.2.3 Summary: Iterative JSON parser with standard Python iterator interfaces Home-page: https://github.com/ICRAR/ijson Author: Rodrigo Tobar, Ivan Sagalaev Author-email: [email protected], [email protected] License: BSD -Description: .. image:: https://travis-ci.com/ICRAR/ijson.svg?branch=master - :target: https://travis-ci.com/ICRAR/ijson +Description: .. image:: https://github.com/ICRAR/ijson/actions/workflows/deploy-to-pypi.yml/badge.svg + :target: https://github.com/ICRAR/ijson/actions/workflows/deploy-to-pypi.yml .. image:: https://ci.appveyor.com/api/projects/status/32wiho6ojw3eakp8/branch/master?svg=true :target: https://ci.appveyor.com/project/rtobar/ijson/branch/master @@ -49,8 +49,8 @@ pip install ijson Binary wheels are provided - for major platforms (Linux, MacOS, Windows) - and python versions (2.7, 3.5+). + for major platforms + and python versions. These are built and published automatically using `cibuildwheel <https://cibuildwheel.readthedocs.io/en/stable/>`_ via Travis CI. @@ -151,6 +151,38 @@ if event == 'map_key' and value == 'name') + .. _command_line: + + Command line + ------------ + + A command line utility is included with ijson + to help visualise the output of each of the routines above. + It reads JSON from the standard input, + and it prints the results of the parsing method chosen by the user + to the standard output. + + The tool is available by running the ``ijson.dump`` module. + For example:: + + $> echo '{"A": 0, "B": [1, 2, 3, 4]}' | python -m ijson.dump -m parse + #: path, name, value + -------------------- + 0: , start_map, None + 1: , map_key, A + 2: A, number, 0 + 3: , map_key, B + 4: B, start_array, None + 5: B.item, number, 1 + 6: B.item, number, 2 + 7: B.item, number, 3 + 8: B.item, number, 4 + 9: B, end_array, None + 10: , end_map, None + + Using ``-h/--help`` will show all available options. + + ``bytes``/``str`` support ------------------------- @@ -224,10 +256,10 @@ parse_events = ijson.parse(io.BytesIO(b'["skip", {"a": 1}, {"b": 2}, {"c": 3}]')) while True: - prefix, event, value = next(parse_event) + prefix, event, value = next(parse_events) if value == "skip": break - for obj in ijson.items(parse_events, 'item') + for obj in ijson.items(parse_events, 'item'): print(obj) @@ -235,6 +267,10 @@ only makes sense for the ``basic_parse -> parse``, ``parse -> items`` and ``parse -> kvitems`` interactions. + Note also that event interception + is currently not supported + by the ``async`` functions. + Push interfaces --------------- @@ -404,12 +440,12 @@ Ijson provides several implementations of the actual parsing in the form of backends located in ijson/backends: - - ``yajl2_c``: a C extension using `YAJL <http://lloyd.github.com/yajl/>`_ 2.x. + - ``yajl2_c``: a C extension using `YAJL <http://lloyd.github.com/yajl/>`__ 2.x. This is the fastest, but *might* require a compiler and the YAJL development files to be present when installing this package. Binary wheel distributions exist for major platforms/architectures to spare users from having to compile the package. - - ``yajl2_cffi``: wrapper around `YAJL <http://lloyd.github.com/yajl/>`_ 2.x + - ``yajl2_cffi``: wrapper around `YAJL <http://lloyd.github.com/yajl/>`__ 2.x using CFFI. - ``yajl2``: wrapper around YAJL 2.x using ctypes, for when you can't use CFFI for some reason. @@ -571,7 +607,7 @@ Python parser in ijson is relatively simple thanks to `Douglas Crockford <http://www.crockford.com/>`_ who invented a strict, easy to parse syntax. - The `YAJL <http://lloyd.github.com/yajl/>`_ library by `Lloyd Hilaiel + The `YAJL <https://lloyd.github.io/yajl>`__ library by `Lloyd Hilaiel <http://lloyd.io/>`_ is the most popular and efficient way to parse JSON in an iterative fashion. @@ -590,6 +626,9 @@ Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Software Development :: Libraries :: Python Modules diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/README.rst new/ijson-3.2.3/README.rst --- old/ijson-3.1.4/README.rst 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/README.rst 2023-07-22 07:17:10.000000000 +0200 @@ -1,5 +1,5 @@ -.. image:: https://travis-ci.com/ICRAR/ijson.svg?branch=master - :target: https://travis-ci.com/ICRAR/ijson +.. image:: https://github.com/ICRAR/ijson/actions/workflows/deploy-to-pypi.yml/badge.svg + :target: https://github.com/ICRAR/ijson/actions/workflows/deploy-to-pypi.yml .. image:: https://ci.appveyor.com/api/projects/status/32wiho6ojw3eakp8/branch/master?svg=true :target: https://ci.appveyor.com/project/rtobar/ijson/branch/master @@ -41,8 +41,8 @@ pip install ijson Binary wheels are provided -for major platforms (Linux, MacOS, Windows) -and python versions (2.7, 3.5+). +for major platforms +and python versions. These are built and published automatically using `cibuildwheel <https://cibuildwheel.readthedocs.io/en/stable/>`_ via Travis CI. @@ -143,6 +143,38 @@ if event == 'map_key' and value == 'name') +.. _command_line: + +Command line +------------ + +A command line utility is included with ijson +to help visualise the output of each of the routines above. +It reads JSON from the standard input, +and it prints the results of the parsing method chosen by the user +to the standard output. + +The tool is available by running the ``ijson.dump`` module. +For example:: + + $> echo '{"A": 0, "B": [1, 2, 3, 4]}' | python -m ijson.dump -m parse + #: path, name, value + -------------------- + 0: , start_map, None + 1: , map_key, A + 2: A, number, 0 + 3: , map_key, B + 4: B, start_array, None + 5: B.item, number, 1 + 6: B.item, number, 2 + 7: B.item, number, 3 + 8: B.item, number, 4 + 9: B, end_array, None + 10: , end_map, None + +Using ``-h/--help`` will show all available options. + + ``bytes``/``str`` support ------------------------- @@ -216,10 +248,10 @@ parse_events = ijson.parse(io.BytesIO(b'["skip", {"a": 1}, {"b": 2}, {"c": 3}]')) while True: - prefix, event, value = next(parse_event) + prefix, event, value = next(parse_events) if value == "skip": break - for obj in ijson.items(parse_events, 'item') + for obj in ijson.items(parse_events, 'item'): print(obj) @@ -227,6 +259,10 @@ only makes sense for the ``basic_parse -> parse``, ``parse -> items`` and ``parse -> kvitems`` interactions. +Note also that event interception +is currently not supported +by the ``async`` functions. + Push interfaces --------------- @@ -396,12 +432,12 @@ Ijson provides several implementations of the actual parsing in the form of backends located in ijson/backends: -- ``yajl2_c``: a C extension using `YAJL <http://lloyd.github.com/yajl/>`_ 2.x. +- ``yajl2_c``: a C extension using `YAJL <http://lloyd.github.com/yajl/>`__ 2.x. This is the fastest, but *might* require a compiler and the YAJL development files to be present when installing this package. Binary wheel distributions exist for major platforms/architectures to spare users from having to compile the package. -- ``yajl2_cffi``: wrapper around `YAJL <http://lloyd.github.com/yajl/>`_ 2.x +- ``yajl2_cffi``: wrapper around `YAJL <http://lloyd.github.com/yajl/>`__ 2.x using CFFI. - ``yajl2``: wrapper around YAJL 2.x using ctypes, for when you can't use CFFI for some reason. @@ -563,7 +599,7 @@ Python parser in ijson is relatively simple thanks to `Douglas Crockford <http://www.crockford.com/>`_ who invented a strict, easy to parse syntax. -The `YAJL <http://lloyd.github.com/yajl/>`_ library by `Lloyd Hilaiel +The `YAJL <https://lloyd.github.io/yajl>`__ library by `Lloyd Hilaiel <http://lloyd.io/>`_ is the most popular and efficient way to parse JSON in an iterative fashion. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/ijson/backends/yajl2_c/async_reading_generator.c new/ijson-3.2.3/ijson/backends/yajl2_c/async_reading_generator.c --- old/ijson-3.1.4/ijson/backends/yajl2_c/async_reading_generator.c 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/ijson/backends/yajl2_c/async_reading_generator.c 2023-07-22 07:17:10.000000000 +0200 @@ -54,6 +54,22 @@ Py_TYPE(self)->tp_free((PyObject*)self); } +static void raise_stopiteration(PyObject *value) +{ +#if defined(PYPY_VERSION) + // PyPy doesn't seem to support normalised exceptions for coroutines, + // see https://foss.heptapod.net/pypy/pypy/-/issues/3965 + PyObject *ex_value = PyObject_CallFunctionObjArgs(PyExc_StopIteration, value, NULL); + PyErr_SetObject(PyExc_StopIteration, ex_value); + Py_DECREF(ex_value); +#else + PyObject *stop_iteration_args = PyTuple_New(1); + PyTuple_SET_ITEM(stop_iteration_args, 0, value); + PyErr_SetObject(PyExc_StopIteration, stop_iteration_args); + Py_DECREF(stop_iteration_args); +#endif +} + static PyObject *maybe_pop_event(async_reading_generator *self) { PyObject *events = self->events; @@ -62,27 +78,42 @@ return NULL; } PyObject *event = PyList_GET_ITEM(events, self->index++); - PyObject *res = PyTuple_New(2); Py_INCREF(event); - PyTuple_SET_ITEM(res, 0, event); - PyErr_SetObject(PyExc_StopIteration, res); - Py_DECREF(res); if (self->index == nevents) { - PySequence_DelSlice(events, 0, self->index); + if (PySequence_DelSlice(events, 0, self->index) == -1) { + Py_RETURN_NONE; + } self->index = 0; } + raise_stopiteration(event); return event; } static int is_gen_coroutine(PyObject *o) { if (PyGen_CheckExact(o)) { - PyCodeObject *code = (PyCodeObject *)((PyGenObject*) o)->gi_code; + PyCodeObject *code = (PyCodeObject *)PyObject_GetAttrString(o, "gi_code"); return code->co_flags & CO_ITERABLE_COROUTINE; } return 0; } +static PyObject* value_from_stopiteration() +{ + PyObject *ptype, *pvalue, *ptraceback, *return_value; + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + if (PyErr_GivenExceptionMatches(pvalue, PyExc_StopIteration)) { + return_value = PyObject_GetAttrString(pvalue, "value"); + Py_XDECREF(pvalue); + } + else { + return_value = pvalue; + } + Py_XDECREF(ptype); + Py_XDECREF(ptraceback); + return return_value; +} + static PyObject *async_reading_generator_next(PyObject *self) { async_reading_generator *gen = (async_reading_generator *)self; @@ -140,19 +171,13 @@ // We await on two things: getting the correct read function (only once), // and reading from it (many times, self->read_func is set) - PyObject *ptype, *ptraceback; if (gen->read_func == NULL) { - PyErr_Fetch(&ptype, &gen->read_func, &ptraceback); - Py_XDECREF(ptype); - Py_XDECREF(ptraceback); + gen->read_func = value_from_stopiteration(); Py_RETURN_NONE; } // Finished awaiting on read() result, parse it - PyObject *buffer; - PyErr_Fetch(&ptype, &buffer, &ptraceback); - Py_XDECREF(ptype); - Py_XDECREF(ptraceback); + PyObject *buffer = value_from_stopiteration(); Py_buffer view; N_M1(PyObject_GetBuffer(buffer, &view, PyBUF_SIMPLE)); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/ijson/backends/yajl2_c/coro_utils.c new/ijson-3.2.3/ijson/backends/yajl2_c/coro_utils.c --- old/ijson-3.1.4/ijson/backends/yajl2_c/coro_utils.c 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/ijson/backends/yajl2_c/coro_utils.c 2023-07-22 07:17:10.000000000 +0200 @@ -14,6 +14,7 @@ PyObject *chain(PyObject *sink, pipeline_node *coro_pipeline) { PyObject *coro = sink; + Py_INCREF(coro); int element = 0; while (1) { pipeline_node node = coro_pipeline[element++]; @@ -24,6 +25,7 @@ if (node.args) { int nargs = PyTuple_Size(node.args); N_N(coro_args = PyTuple_New(nargs + 1)); + Py_INCREF(coro); PyTuple_SET_ITEM(coro_args, 0, coro); int i; for (i = 0; i != nargs; i++) { @@ -32,8 +34,8 @@ } else { N_N(coro_args = PyTuple_Pack(1, coro)); - Py_DECREF(coro); } + Py_DECREF(coro); N_N(coro = PyObject_Call((PyObject *)node.type, coro_args, node.kwargs)); Py_DECREF(coro_args); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/ijson/backends/yajl2_c/kvitems_basecoro.c new/ijson-3.2.3/ijson/backends/yajl2_c/kvitems_basecoro.c --- old/ijson-3.1.4/ijson/backends/yajl2_c/kvitems_basecoro.c 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/ijson/backends/yajl2_c/kvitems_basecoro.c 2023-07-22 07:17:10.000000000 +0200 @@ -119,7 +119,7 @@ #endif .tp_basicsize = sizeof(KVItemsBasecoro), .tp_name = "_yajl2.kvitems_basecoro", - .tp_doc = "Coroutine dispathing (key, value) tuples", + .tp_doc = "Coroutine dispatching (key, value) tuples", .tp_init = (initproc)kvitems_basecoro_init, .tp_dealloc = (destructor)kvitems_basecoro_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/ijson/backends/yajl2_c/parse_basecoro.c new/ijson-3.2.3/ijson/backends/yajl2_c/parse_basecoro.c --- old/ijson-3.1.4/ijson/backends/yajl2_c/parse_basecoro.c 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/ijson/backends/yajl2_c/parse_basecoro.c 2023-07-22 07:17:10.000000000 +0200 @@ -13,6 +13,11 @@ #include "kvitems_basecoro.h" #include "parse_basecoro.h" +#if PY_MAJOR_VERSION >= 3 +#define ijson_unicode_length PyUnicode_GET_LENGTH +#else +#define ijson_unicode_length PyUnicode_GET_SIZE +#endif /* * __init__, destructor, __iter__ and __next__ @@ -90,7 +95,7 @@ PyObject *last_path; N_N(last_path = PySequence_GetItem(gen->path, npaths - 1)); - if (PyUnicode_GET_SIZE(last_path) > 0) { + if (ijson_unicode_length(last_path) > 0) { PyObject *new_path; CONCAT(new_path, last_path, dotitem); N_M1(PyList_Append(gen->path, new_path)); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/ijson/backends/yajl2_c/reading_generator.c new/ijson-3.2.3/ijson/backends/yajl2_c/reading_generator.c --- old/ijson-3.1.4/ijson/backends/yajl2_c/reading_generator.c 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/ijson/backends/yajl2_c/reading_generator.c 2023-07-22 07:17:10.000000000 +0200 @@ -74,6 +74,7 @@ length = view.len; PyObject *send_res = ijson_yajl_parse(basic_parse_basecoro->h, view.buf, view.len); Py_DECREF(pbuffer); + PyBuffer_Release(&view); N_N(send_res); } else { @@ -84,9 +85,10 @@ N_M1(length); Py_DECREF(plength); N_M1(PyObject_GetBuffer(self->buffer, &view, PyBUF_SIMPLE)); - N_N(ijson_yajl_parse(basic_parse_basecoro->h, view.buf, length)); + PyObject *send_res = ijson_yajl_parse(basic_parse_basecoro->h, view.buf, length); + PyBuffer_Release(&view); + N_N(send_res); } - PyBuffer_Release(&view); nevents = PyList_Size(events); if (length == 0) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/ijson/dump.py new/ijson-3.2.3/ijson/dump.py --- old/ijson-3.1.4/ijson/dump.py 1970-01-01 01:00:00.000000000 +0100 +++ new/ijson-3.2.3/ijson/dump.py 2023-07-22 07:17:10.000000000 +0200 @@ -0,0 +1,57 @@ +'''Dumping command-line utility''' + +import argparse +import sys + +import ijson +from . import compat + + +HEADERS = { + 'basic_parse': 'name, value', + 'parse': 'path, name, value', + 'kvitems': 'key, value', + 'items': 'value', +} + +def to_string(o): + if isinstance(o, compat.texttype) and compat.IS_PY2: + o = o.encode('utf8') + if isinstance(o, compat.bytetype): + return compat.b2s(o) + return str(o) + +def dump(): + parser = argparse.ArgumentParser(description='Dump ijson events') + parser.add_argument('-m', '--method', choices=['basic_parse', 'parse', 'kvitems', 'items'], + help='The method to use for dumping', default='basic_parse') + parser.add_argument('-p', '--prefix', help='Prefix (used with -M items|kvitems)', default='') + parser.add_argument('-M', '--multiple-values', help='Allow multiple values', action='store_true') + args = parser.parse_args() + + method = getattr(ijson, args.method) + method_args = () + method_kwargs = {} + if args.method in ('items', 'kvitems'): + method_args = args.prefix, + if args.multiple_values: + method_kwargs['multiple_values'] = True + header = '#: ' + HEADERS[args.method] + print(header) + print('-' * len(header)) + + # Use the raw bytes stream in stdin if possible + stdin = sys.stdin + if hasattr(stdin, 'buffer'): + stdin = stdin.buffer + + enumerated_results = enumerate(method(stdin, *method_args, **method_kwargs)) + if args.method == 'items': + for i, result in enumerated_results: + print('%i: %s' % (i, result)) + else: + for i, result in enumerated_results: + print('%i: %s' % (i, ', '.join(to_string(bit) for bit in result))) + +if __name__ == '__main__': + dump() \ No newline at end of file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/ijson/version.py new/ijson-3.2.3/ijson/version.py --- old/ijson-3.1.4/ijson/version.py 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/ijson/version.py 2023-07-22 07:17:10.000000000 +0200 @@ -1 +1 @@ -__version__ = '3.1.4' +__version__ = '3.2.3' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/ijson.egg-info/PKG-INFO new/ijson-3.2.3/ijson.egg-info/PKG-INFO --- old/ijson-3.1.4/ijson.egg-info/PKG-INFO 2021-03-02 11:05:01.000000000 +0100 +++ new/ijson-3.2.3/ijson.egg-info/PKG-INFO 2023-07-22 07:17:12.000000000 +0200 @@ -1,13 +1,13 @@ Metadata-Version: 2.1 Name: ijson -Version: 3.1.4 +Version: 3.2.3 Summary: Iterative JSON parser with standard Python iterator interfaces Home-page: https://github.com/ICRAR/ijson Author: Rodrigo Tobar, Ivan Sagalaev Author-email: [email protected], [email protected] License: BSD -Description: .. image:: https://travis-ci.com/ICRAR/ijson.svg?branch=master - :target: https://travis-ci.com/ICRAR/ijson +Description: .. image:: https://github.com/ICRAR/ijson/actions/workflows/deploy-to-pypi.yml/badge.svg + :target: https://github.com/ICRAR/ijson/actions/workflows/deploy-to-pypi.yml .. image:: https://ci.appveyor.com/api/projects/status/32wiho6ojw3eakp8/branch/master?svg=true :target: https://ci.appveyor.com/project/rtobar/ijson/branch/master @@ -49,8 +49,8 @@ pip install ijson Binary wheels are provided - for major platforms (Linux, MacOS, Windows) - and python versions (2.7, 3.5+). + for major platforms + and python versions. These are built and published automatically using `cibuildwheel <https://cibuildwheel.readthedocs.io/en/stable/>`_ via Travis CI. @@ -151,6 +151,38 @@ if event == 'map_key' and value == 'name') + .. _command_line: + + Command line + ------------ + + A command line utility is included with ijson + to help visualise the output of each of the routines above. + It reads JSON from the standard input, + and it prints the results of the parsing method chosen by the user + to the standard output. + + The tool is available by running the ``ijson.dump`` module. + For example:: + + $> echo '{"A": 0, "B": [1, 2, 3, 4]}' | python -m ijson.dump -m parse + #: path, name, value + -------------------- + 0: , start_map, None + 1: , map_key, A + 2: A, number, 0 + 3: , map_key, B + 4: B, start_array, None + 5: B.item, number, 1 + 6: B.item, number, 2 + 7: B.item, number, 3 + 8: B.item, number, 4 + 9: B, end_array, None + 10: , end_map, None + + Using ``-h/--help`` will show all available options. + + ``bytes``/``str`` support ------------------------- @@ -224,10 +256,10 @@ parse_events = ijson.parse(io.BytesIO(b'["skip", {"a": 1}, {"b": 2}, {"c": 3}]')) while True: - prefix, event, value = next(parse_event) + prefix, event, value = next(parse_events) if value == "skip": break - for obj in ijson.items(parse_events, 'item') + for obj in ijson.items(parse_events, 'item'): print(obj) @@ -235,6 +267,10 @@ only makes sense for the ``basic_parse -> parse``, ``parse -> items`` and ``parse -> kvitems`` interactions. + Note also that event interception + is currently not supported + by the ``async`` functions. + Push interfaces --------------- @@ -404,12 +440,12 @@ Ijson provides several implementations of the actual parsing in the form of backends located in ijson/backends: - - ``yajl2_c``: a C extension using `YAJL <http://lloyd.github.com/yajl/>`_ 2.x. + - ``yajl2_c``: a C extension using `YAJL <http://lloyd.github.com/yajl/>`__ 2.x. This is the fastest, but *might* require a compiler and the YAJL development files to be present when installing this package. Binary wheel distributions exist for major platforms/architectures to spare users from having to compile the package. - - ``yajl2_cffi``: wrapper around `YAJL <http://lloyd.github.com/yajl/>`_ 2.x + - ``yajl2_cffi``: wrapper around `YAJL <http://lloyd.github.com/yajl/>`__ 2.x using CFFI. - ``yajl2``: wrapper around YAJL 2.x using ctypes, for when you can't use CFFI for some reason. @@ -571,7 +607,7 @@ Python parser in ijson is relatively simple thanks to `Douglas Crockford <http://www.crockford.com/>`_ who invented a strict, easy to parse syntax. - The `YAJL <http://lloyd.github.com/yajl/>`_ library by `Lloyd Hilaiel + The `YAJL <https://lloyd.github.io/yajl>`__ library by `Lloyd Hilaiel <http://lloyd.io/>`_ is the most popular and efficient way to parse JSON in an iterative fashion. @@ -590,6 +626,9 @@ Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Software Development :: Libraries :: Python Modules diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/ijson.egg-info/SOURCES.txt new/ijson-3.2.3/ijson.egg-info/SOURCES.txt --- old/ijson-3.1.4/ijson.egg-info/SOURCES.txt 2021-03-02 11:05:01.000000000 +0100 +++ new/ijson-3.2.3/ijson.egg-info/SOURCES.txt 2023-07-22 07:17:12.000000000 +0200 @@ -7,6 +7,7 @@ ijson/__init__.py ijson/common.py ijson/compat.py +ijson/dump.py ijson/utils.py ijson/utils35.py ijson/version.py @@ -62,5 +63,6 @@ test/test_async_types_coroutines.py test/test_base.py test/test_coroutines.py +test/test_dump.py test/test_generators.py test/test_misc.py \ No newline at end of file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/setup.py new/ijson-3.2.3/setup.py --- old/ijson-3.1.4/setup.py 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/setup.py 2023-07-22 07:17:10.000000000 +0200 @@ -1,5 +1,10 @@ -import distutils.ccompiler -import distutils.sysconfig +try: + from distutils import ccompiler + from distutils import sysconfig +except ImportError: + from setuptools._distutils import ccompiler + from setuptools._distutils import sysconfig + import glob import os import platform @@ -37,6 +42,9 @@ 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Software Development :: Libraries :: Python Modules', @@ -49,8 +57,8 @@ # without properly cleaning up def yajl_present(): - compiler = distutils.ccompiler.new_compiler(verbose=1) - distutils.sysconfig.customize_compiler(compiler) # CC, CFLAGS, LDFLAGS, etc + compiler = ccompiler.new_compiler(verbose=1) + sysconfig.customize_compiler(compiler) # CC, CFLAGS, LDFLAGS, etc fname = tempfile.mktemp(".c", "yajl_version") try: @@ -109,7 +117,9 @@ extra_sources.remove(os.path.join(yajl_sources, 'src', 'yajl_version.c')) extra_include_dirs = [yajl_sources, os.path.join(yajl_sources, 'src')] libs = [] -if embed_yajl or have_yajl: +build_yajl_default = '1' if embed_yajl or have_yajl else '0' +build_yajl = os.environ.get('IJSON_BUILD_YAJL2C', build_yajl_default) == '1' +if build_yajl: yajl_ext = Extension('ijson.backends._yajl2', language='c', sources=sorted(glob.glob('ijson/backends/yajl2_c/*.c')) + extra_sources, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/test/_test_async.py new/ijson-3.2.3/test/_test_async.py --- old/ijson-3.1.4/test/_test_async.py 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/test/_test_async.py 2023-07-22 07:17:10.000000000 +0200 @@ -1,7 +1,6 @@ # -*- coding:utf-8 -*- import asyncio -import contextlib import io from ijson import compat diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/test/_test_async_common.py new/ijson-3.2.3/test/_test_async_common.py --- old/ijson-3.1.4/test/_test_async_common.py 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/test/_test_async_common.py 2023-07-22 07:17:10.000000000 +0200 @@ -2,9 +2,6 @@ import asyncio import contextlib -import io - -from ijson import compat def _aiorun(f): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/test/_test_async_types_coroutine.py new/ijson-3.2.3/test/_test_async_types_coroutine.py --- old/ijson-3.1.4/test/_test_async_types_coroutine.py 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/test/_test_async_types_coroutine.py 2023-07-22 07:17:10.000000000 +0200 @@ -1,7 +1,5 @@ # -*- coding:utf-8 -*- -import asyncio -import contextlib import io import types diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/test/test_dump.py new/ijson-3.2.3/test/test_dump.py --- old/ijson-3.1.4/test/test_dump.py 1970-01-01 01:00:00.000000000 +0100 +++ new/ijson-3.2.3/test/test_dump.py 2023-07-22 07:17:10.000000000 +0200 @@ -0,0 +1,46 @@ +import os +import subprocess +import sys +import unittest + +from ijson import compat +from test.test_base import JSON + + +class DumpTests(unittest.TestCase): + + def _do_test_dump(self, method, multiple_values): + # Use python backend to ensure multiple_values works + env = dict(os.environ) + env['IJSON_BACKEND'] = 'python' + # Ensure printing works on the subprocess in Windows + # by using utf-8 on its stdout + if 'win' in sys.platform: + env = dict(os.environ) + env['PYTHONIOENCODING'] = 'utf-8' + cmd = [sys.executable, '-m', 'ijson.dump', '-m', method, '-p', ''] + if multiple_values: + cmd.append('-M') + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + input_data = JSON + if multiple_values: + input_data += JSON + out, err = proc.communicate(input_data) + status = proc.wait() + self.assertEqual(0, status, "out:\n%s\nerr:%s" % (compat.b2s(out), compat.b2s(err))) + + def _test_dump(self, method): + self._do_test_dump(method, True) + self._do_test_dump(method, False) + + def test_basic_parse(self): + self._test_dump('basic_parse') + + def test_parse(self): + self._test_dump('parse') + + def test_kvitems(self): + self._test_dump('kvitems') + + def test_items(self): + self._test_dump('items') \ No newline at end of file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/ijson-3.1.4/test/test_misc.py new/ijson-3.2.3/test/test_misc.py --- old/ijson-3.1.4/test/test_misc.py 2021-03-02 11:04:56.000000000 +0100 +++ new/ijson-3.2.3/test/test_misc.py 2023-07-22 07:17:10.000000000 +0200 @@ -1,3 +1,4 @@ +import importlib.util import unittest from ijson import common, compat @@ -8,7 +9,7 @@ class Misc(unittest.TestCase): - """Miscelaneous unit tests""" + """Miscellaneous unit tests""" def test_common_number_is_deprecated(self): with warning_catcher() as warns: @@ -16,6 +17,15 @@ self.assertEqual(len(warns), 1) self.assertEqual(DeprecationWarning, warns[0].category) + def test_yajl2_c_loadable(self): + if compat.IS_PY2: + self.skipTest("Test requires Python 3.4+") + spec = importlib.util.find_spec("ijson.backends._yajl2") + if spec is None: + self.skipTest("yajl2_c is not built") + importlib.util.module_from_spec(spec) + + class MainEntryPoints(object): def _assert_invalid_type(self, routine, *args, **kwargs):
