Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-zarr for openSUSE:Factory checked in at 2021-09-01 21:37:16 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-zarr (Old) and /work/SRC/openSUSE:Factory/.python-zarr.new.1899 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-zarr" Wed Sep 1 21:37:16 2021 rev:5 rq:915459 version:2.9.5 Changes: -------- --- /work/SRC/openSUSE:Factory/python-zarr/python-zarr.changes 2021-08-28 22:29:08.313975984 +0200 +++ /work/SRC/openSUSE:Factory/.python-zarr.new.1899/python-zarr.changes 2021-09-01 21:37:38.676910368 +0200 @@ -1,0 +2,9 @@ +Wed Sep 1 09:34:16 UTC 2021 - Matej Cepl <mc...@suse.com> + +- Update to 2.9.5: + - Fix FSStore.listdir behavior for nested directories (replaces + now unnecessary skip-825-TestFSStore-test_create.patch). + - Fix structured arrays that contain objects (which replaces + zarr-pr802-fsspec-2021-07.patch). + +------------------------------------------------------------------- Old: ---- skip-825-TestFSStore-test_create.patch zarr-2.9.3.tar.gz zarr-pr802-fsspec-2021-07.patch New: ---- zarr-2.9.5.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-zarr.spec ++++++ --- /var/tmp/diff_new_pack.Rw5MzB/_old 2021-09-01 21:37:39.256911066 +0200 +++ /var/tmp/diff_new_pack.Rw5MzB/_new 2021-09-01 21:37:39.260911070 +0200 @@ -20,17 +20,12 @@ %define skip_python2 1 %define skip_python36 1 Name: python-zarr -Version: 2.9.3 +Version: 2.9.5 Release: 0 Summary: An implementation of chunked, compressed, N-dimensional arrays for Python License: MIT URL: https://github.com/zarr-developers/zarr-python Source: https://files.pythonhosted.org/packages/source/z/zarr/zarr-%{version}.tar.gz -# PATCH-FIX-UPSTREAM zarr-pr802-fsspec-2021-07.patch -- gh#zarr-developers/zarr-python#802 and gh#zarr-developers/zarr-python#812 -Patch0: zarr-pr802-fsspec-2021-07.patch -# PATCH-FIX-UPSTREAM skip-825-TestFSStore-test_create.patch gh#zarr-developers/zarr-python#825 mc...@suse.com -# Skip the failing test -Patch1: skip-825-TestFSStore-test_create.patch BuildRequires: %{python_module setuptools >= 38.6.0} BuildRequires: %{python_module setuptools_scm} BuildRequires: fdupes ++++++ zarr-2.9.3.tar.gz -> zarr-2.9.5.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/PKG-INFO new/zarr-2.9.5/PKG-INFO --- old/zarr-2.9.3/PKG-INFO 2021-08-26 11:57:52.194819700 +0200 +++ new/zarr-2.9.5/PKG-INFO 2021-09-01 10:21:28.011391600 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: zarr -Version: 2.9.3 +Version: 2.9.5 Summary: An implementation of chunked, compressed, N-dimensional arrays for Python. Home-page: https://github.com/zarr-developers/zarr-python Maintainer: Alistair Miles diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/docs/release.rst new/zarr-2.9.5/docs/release.rst --- old/zarr-2.9.3/docs/release.rst 2021-08-26 11:57:39.000000000 +0200 +++ new/zarr-2.9.5/docs/release.rst 2021-09-01 10:21:18.000000000 +0200 @@ -6,6 +6,28 @@ Unreleased ---------- +.. _release_2.9.5: + +2.9.5 +----- + +Bug fixes +~~~~~~~~~ + +* Fix FSStore.listdir behavior for nested directories. + By :user:`Greggory Lee <grlee77>`; :issue:`802`. + +.. _release_2.9.4: + +2.9.4 +----- + +Bug fixes +~~~~~~~~~ + +* Fix structured arrays that contain objects + By :user: `Attila Bergou <abergou>`; :issue: `806` + .. _release_2.9.3: 2.9.3 @@ -31,7 +53,6 @@ * Correct conda-forge deployment of Zarr by fixing some Zarr tests. By :user:`Ben Williams <benjaminhwilliams>`; :issue:`821`. - .. _release_2.9.1: 2.9.1 @@ -92,7 +113,6 @@ * TST: add missing assert in test_hexdigest. By :user:`Greggory Lee <grlee77>`; :issue:`801`. - .. _release_2.8.3: 2.8.3 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/requirements_dev_minimal.txt new/zarr-2.9.5/requirements_dev_minimal.txt --- old/zarr-2.9.3/requirements_dev_minimal.txt 2021-08-26 11:57:40.000000000 +0200 +++ new/zarr-2.9.5/requirements_dev_minimal.txt 2021-09-01 10:21:18.000000000 +0200 @@ -5,4 +5,4 @@ msgpack-python==0.5.6 setuptools-scm==6.0.1 # test requirements -pytest==6.2.4 +pytest==6.2.5 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/requirements_dev_optional.txt new/zarr-2.9.5/requirements_dev_optional.txt --- old/zarr-2.9.3/requirements_dev_optional.txt 2021-08-26 11:57:40.000000000 +0200 +++ new/zarr-2.9.5/requirements_dev_optional.txt 2021-09-01 10:21:18.000000000 +0200 @@ -9,6 +9,7 @@ azure-storage-blob==12.8.1 # pyup: ignore redis==3.5.3 types-redis +types-setuptools pymongo==3.12.0 # optional test requirements tox==3.24.3 @@ -17,7 +18,5 @@ pytest-cov==2.12.1 pytest-doctestplus==0.10.1 h5py==3.4.0 -s3fs==2021.6.0 -fsspec==2021.6.0 -aiobotocore!=1.4.0 +fsspec[s3]==2021.08.1 moto[server]>=1.3.14 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/zarr/codecs.py new/zarr-2.9.5/zarr/codecs.py --- old/zarr-2.9.3/zarr/codecs.py 2021-08-26 11:57:40.000000000 +0200 +++ new/zarr-2.9.5/zarr/codecs.py 2021-09-01 10:21:18.000000000 +0200 @@ -1,4 +1,4 @@ # flake8: noqa from numcodecs import * -from numcodecs import get_codec, Blosc, Zlib, Delta, AsType, BZ2 +from numcodecs import get_codec, Blosc, Pickle, Zlib, Delta, AsType, BZ2 from numcodecs.registry import codec_registry diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/zarr/meta.py new/zarr-2.9.5/zarr/meta.py --- old/zarr-2.9.3/zarr/meta.py 2021-08-26 11:57:40.000000000 +0200 +++ new/zarr-2.9.5/zarr/meta.py 2021-09-01 10:21:18.000000000 +0200 @@ -40,7 +40,14 @@ # extract array metadata fields try: dtype = decode_dtype(meta['dtype']) - fill_value = decode_fill_value(meta['fill_value'], dtype) + + if dtype.hasobject: + import numcodecs + object_codec = numcodecs.get_codec(meta['filters'][0]) + else: + object_codec = None + + fill_value = decode_fill_value(meta['fill_value'], dtype, object_codec) meta = dict( zarr_format=meta['zarr_format'], shape=tuple(meta['shape']), @@ -66,14 +73,18 @@ dtype, sdshape = dtype.subdtype dimension_separator = meta.get('dimension_separator') - + if dtype.hasobject: + import numcodecs + object_codec = numcodecs.get_codec(meta['filters'][0]) + else: + object_codec = None meta = dict( zarr_format=ZARR_FORMAT, shape=meta['shape'] + sdshape, chunks=meta['chunks'], dtype=encode_dtype(dtype), compressor=meta['compressor'], - fill_value=encode_fill_value(meta['fill_value'], dtype), + fill_value=encode_fill_value(meta['fill_value'], dtype, object_codec), order=meta['order'], filters=meta['filters'], ) @@ -132,10 +143,17 @@ } -def decode_fill_value(v, dtype): +def decode_fill_value(v, dtype, object_codec=None): # early out if v is None: return v + if dtype.kind == 'V' and dtype.hasobject: + if object_codec is None: + raise ValueError('missing object_codec for object array') + v = base64.standard_b64decode(v) + v = object_codec.decode(v) + v = np.array(v, dtype=dtype)[()] + return v if dtype.kind == 'f': if v == 'NaN': return np.nan @@ -171,10 +189,16 @@ return np.array(v, dtype=dtype)[()] -def encode_fill_value(v: Any, dtype: np.dtype) -> Any: +def encode_fill_value(v: Any, dtype: np.dtype, object_codec: Any = None) -> Any: # early out if v is None: return v + if dtype.kind == 'V' and dtype.hasobject: + if object_codec is None: + raise ValueError('missing object_codec for object array') + v = object_codec.encode(v) + v = str(base64.standard_b64encode(v), 'ascii') + return v if dtype.kind == 'f': if np.isnan(v): return 'NaN' @@ -190,8 +214,8 @@ return bool(v) elif dtype.kind in 'c': c = cast(np.complex128, np.dtype(complex).type()) - v = (encode_fill_value(v.real, c.real.dtype), - encode_fill_value(v.imag, c.imag.dtype)) + v = (encode_fill_value(v.real, c.real.dtype, object_codec), + encode_fill_value(v.imag, c.imag.dtype, object_codec)) return v elif dtype.kind in 'SV': v = str(base64.standard_b64encode(v), 'ascii') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/zarr/storage.py new/zarr-2.9.5/zarr/storage.py --- old/zarr-2.9.3/zarr/storage.py 2021-08-26 11:57:40.000000000 +0200 +++ new/zarr-2.9.5/zarr/storage.py 2021-09-01 10:21:18.000000000 +0200 @@ -423,7 +423,7 @@ filters_config = [] # deal with object encoding - if dtype == object: + if dtype.hasobject: if object_codec is None: if not filters: # there are no filters so we can be sure there is no object codec @@ -1189,7 +1189,8 @@ for file_name in self.fs.find(entry_path): file_path = os.path.join(dir_path, file_name) rel_path = file_path.split(root_path)[1] - new_children.append(rel_path.replace(os.path.sep, '.')) + rel_path = rel_path.lstrip('/') + new_children.append(rel_path.replace('/', '.')) else: new_children.append(entry) return sorted(new_children) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/zarr/tests/test_core.py new/zarr-2.9.5/zarr/tests/test_core.py --- old/zarr-2.9.3/zarr/tests/test_core.py 2021-08-26 11:57:40.000000000 +0200 +++ new/zarr-2.9.5/zarr/tests/test_core.py 2021-09-01 10:21:18.000000000 +0200 @@ -15,6 +15,7 @@ from numcodecs.compat import ensure_bytes, ensure_ndarray from numcodecs.tests.common import greetings from numpy.testing import assert_array_almost_equal, assert_array_equal +from pkg_resources import parse_version from zarr.core import Array from zarr.meta import json_loads @@ -1362,6 +1363,44 @@ if hasattr(z.store, 'close'): z.store.close() + @unittest.skipIf(parse_version(np.__version__) < parse_version('1.14.0'), + "unsupported numpy version") + def test_structured_array_contain_object(self): + + if "PartialRead" in self.__class__.__name__: + pytest.skip("partial reads of object arrays not supported") + + # ----------- creation -------------- + + structured_dtype = [('c_obj', object), ('c_int', int)] + a = np.array([(b'aaa', 1), + (b'bbb', 2)], dtype=structured_dtype) + + # zarr-array with structured dtype require object codec + with pytest.raises(ValueError): + self.create_array(shape=a.shape, dtype=structured_dtype) + + # create zarr-array by np-array + za = self.create_array(shape=a.shape, dtype=structured_dtype, object_codec=Pickle()) + za[:] = a + + # must be equal + assert_array_equal(a, za[:]) + + # ---------- indexing --------------- + + assert za[0] == a[0] + + za[0] = (b'ccc', 3) + za[1:2] = np.array([(b'ddd', 4)], dtype=structured_dtype) # ToDo: not work with list + assert_array_equal(za[:], np.array([(b'ccc', 3), (b'ddd', 4)], dtype=structured_dtype)) + + za['c_obj'] = [b'eee', b'fff'] + za['c_obj', 0] = b'ggg' + assert_array_equal(za[:], np.array([(b'ggg', 3), (b'fff', 4)], dtype=structured_dtype)) + assert za['c_obj', 0] == b'ggg' + assert za[1, 'c_int'] == 4 + def test_iteration_exceptions(self): # zero d array a = np.array(1, dtype=int) @@ -1490,6 +1529,14 @@ if hasattr(a.store, 'close'): a.store.close() + def test_structured_with_object(self): + a = self.create_array(fill_value=(0.0, None), + shape=10, + chunks=10, + dtype=[('x', float), ('y', object)], + object_codec=Pickle()) + assert tuple(a[0]) == (0.0, None) + class TestArrayWithPath(TestArray): @@ -1893,6 +1940,14 @@ # Cannot hacking out object codec as N5 doesn't allow object codecs pass + def test_structured_with_object(self): + # Cannot hacking out object codec as N5 doesn't allow object codecs + pass + + def test_structured_array_contain_object(self): + # Cannot hacking out object codec as N5 doesn't allow object codecs + pass + def test_attrs_n5_keywords(self): z = self.create_array(shape=(1050,), chunks=100, dtype='i4') for k in n5_keywords: @@ -2326,6 +2381,10 @@ # skip this one, cannot use delta with objects pass + def test_structured_array_contain_object(self): + # skip this one, cannot use delta on structured array + pass + # custom store, does not support getsize() class CustomMapping(object): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/zarr/tests/test_meta.py new/zarr-2.9.5/zarr/tests/test_meta.py --- old/zarr-2.9.3/zarr/tests/test_meta.py 2021-08-26 11:57:40.000000000 +0200 +++ new/zarr-2.9.5/zarr/tests/test_meta.py 2021-09-01 10:21:18.000000000 +0200 @@ -4,11 +4,12 @@ import numpy as np import pytest -from zarr.codecs import Blosc, Delta, Zlib +from zarr.codecs import Blosc, Delta, Pickle, Zlib from zarr.errors import MetadataError from zarr.meta import (ZARR_FORMAT, decode_array_metadata, decode_dtype, decode_group_metadata, encode_array_metadata, - encode_dtype) + encode_dtype, encode_fill_value, decode_fill_value) +from zarr.util import normalize_dtype, normalize_fill_value def assert_json_equal(expect, actual): @@ -435,3 +436,69 @@ }''' % (ZARR_FORMAT - 1) with pytest.raises(MetadataError): decode_group_metadata(b) + + +@pytest.mark.parametrize( + "fill_value,dtype,object_codec,result", + [ + ( + (0.0, None), + [('x', float), ('y', object)], + Pickle(), + True, # Pass + ), + ( + (0.0, None), + [('x', float), ('y', object)], + None, + False, # Fail + ), + ], +) +def test_encode_fill_value(fill_value, dtype, object_codec, result): + + # normalize metadata (copied from _init_array_metadata) + dtype, object_codec = normalize_dtype(dtype, object_codec) + dtype = dtype.base + fill_value = normalize_fill_value(fill_value, dtype) + + # test + if result: + encode_fill_value(fill_value, dtype, object_codec) + else: + with pytest.raises(ValueError): + encode_fill_value(fill_value, dtype, object_codec) + + +@pytest.mark.parametrize( + "fill_value,dtype,object_codec,result", + [ + ( + (0.0, None), + [('x', float), ('y', object)], + Pickle(), + True, # Pass + ), + ( + (0.0, None), + [('x', float), ('y', object)], + None, + False, # Fail + ), + ], +) +def test_decode_fill_value(fill_value, dtype, object_codec, result): + + # normalize metadata (copied from _init_array_metadata) + dtype, object_codec = normalize_dtype(dtype, object_codec) + dtype = dtype.base + fill_value = normalize_fill_value(fill_value, dtype) + + # test + if result: + v = encode_fill_value(fill_value, dtype, object_codec) + decode_fill_value(v, dtype, object_codec) + else: + with pytest.raises(ValueError): + # No encoding is possible + decode_fill_value(fill_value, dtype, object_codec) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/zarr/util.py new/zarr-2.9.5/zarr/util.py --- old/zarr-2.9.3/zarr/util.py 2021-08-26 11:57:40.000000000 +0200 +++ new/zarr-2.9.5/zarr/util.py 2021-09-01 10:21:18.000000000 +0200 @@ -253,10 +253,9 @@ def normalize_fill_value(fill_value, dtype: np.dtype): - if fill_value is None: + if fill_value is None or dtype.hasobject: # no fill value pass - elif fill_value == 0: # this should be compatible across numpy versions for any array type, including # structured arrays diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/zarr/version.py new/zarr-2.9.5/zarr/version.py --- old/zarr-2.9.3/zarr/version.py 2021-08-26 11:57:49.000000000 +0200 +++ new/zarr-2.9.5/zarr/version.py 2021-09-01 10:21:26.000000000 +0200 @@ -1,5 +1,5 @@ # coding: utf-8 # file generated by setuptools_scm # don't change, don't track in version control -version = '2.9.3' -version_tuple = (2, 9, 3) +version = '2.9.5' +version_tuple = (2, 9, 5) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/zarr-2.9.3/zarr.egg-info/PKG-INFO new/zarr-2.9.5/zarr.egg-info/PKG-INFO --- old/zarr-2.9.3/zarr.egg-info/PKG-INFO 2021-08-26 11:57:49.000000000 +0200 +++ new/zarr-2.9.5/zarr.egg-info/PKG-INFO 2021-09-01 10:21:26.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: zarr -Version: 2.9.3 +Version: 2.9.5 Summary: An implementation of chunked, compressed, N-dimensional arrays for Python. Home-page: https://github.com/zarr-developers/zarr-python Maintainer: Alistair Miles
