Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package python-zarr for openSUSE:Factory
checked in at 2024-04-21 20:26:43
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-zarr (Old)
and /work/SRC/openSUSE:Factory/.python-zarr.new.26366 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-zarr"
Sun Apr 21 20:26:43 2024 rev:13 rq:1169325 version:2.17.2
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-zarr/python-zarr.changes 2024-03-19
17:31:15.081469728 +0100
+++ /work/SRC/openSUSE:Factory/.python-zarr.new.26366/python-zarr.changes
2024-04-21 20:27:53.959282464 +0200
@@ -1,0 +2,16 @@
+Sat Apr 20 08:12:17 UTC 2024 - Dirk Müller <dmuel...@suse.com>
+
+- update to 2.17.2:
+ * [v3] Dramatically reduce number of __contains__ requests in
+ favor of optimistically calling __getitem__ and handling any
+ error that may arise.
+ * [v3] Reuse the downloaded array metadata when creating an
+ Array.
+ * Optimize Array.info so that it calls getsize only once.
+ * Override IPython _repr_*_ methods to avoid expensive lookups
+ against object stores.
+ * FSStore now raises rather than return bad data.
+ * Add CI test environment for Python 3.12
+ * Bump minimum supported NumPy version to 1.23 (per spec 0000)
+
+-------------------------------------------------------------------
Old:
----
zarr-2.17.1.tar.gz
New:
----
zarr-2.17.2.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-zarr.spec ++++++
--- /var/tmp/diff_new_pack.oy2GIu/_old 2024-04-21 20:27:54.751311526 +0200
+++ /var/tmp/diff_new_pack.oy2GIu/_new 2024-04-21 20:27:54.751311526 +0200
@@ -18,7 +18,7 @@
%{?sle15_python_module_pythons}
Name: python-zarr
-Version: 2.17.1
+Version: 2.17.2
Release: 0
Summary: An implementation of chunked, compressed, N-dimensional arrays
for Python
License: MIT
@@ -37,7 +37,7 @@
Requires: python-asciitree
Requires: python-fasteners
Requires: python-numcodecs >= 0.10.0
-Requires: python-numpy >= 1.21.1
+Requires: python-numpy >= 1.23
Suggests: python-dbm
Suggests: python-ipytree
Suggests: python-msgpack
@@ -49,7 +49,7 @@
BuildRequires: %{python_module fasteners}
BuildRequires: %{python_module msgpack}
BuildRequires: %{python_module numcodecs >= 0.10.0}
-BuildRequires: %{python_module numpy >= 1.21.1}
+BuildRequires: %{python_module numpy >= 1.23}
BuildRequires: %{python_module pytest}
# /SECTION
%python_subpackages
++++++ zarr-2.17.1.tar.gz -> zarr-2.17.2.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/.github/workflows/minimal.yml
new/zarr-2.17.2/.github/workflows/minimal.yml
--- old/zarr-2.17.1/.github/workflows/minimal.yml 2024-03-06
20:26:17.000000000 +0100
+++ new/zarr-2.17.2/.github/workflows/minimal.yml 2024-04-05
22:55:46.000000000 +0200
@@ -15,7 +15,7 @@
steps:
- uses: actions/checkout@v4
- name: Setup Miniconda
- uses: conda-incubator/setup-miniconda@v3.0.2
+ uses: conda-incubator/setup-miniconda@v3.0.3
with:
channels: conda-forge
environment-file: environment.yml
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/.github/workflows/python-package.yml
new/zarr-2.17.2/.github/workflows/python-package.yml
--- old/zarr-2.17.1/.github/workflows/python-package.yml 2024-03-06
20:26:17.000000000 +0100
+++ new/zarr-2.17.2/.github/workflows/python-package.yml 2024-04-05
22:55:46.000000000 +0200
@@ -15,13 +15,15 @@
runs-on: ubuntu-latest
strategy:
matrix:
- python-version: ['3.9', '3.10', '3.11']
- numpy_version: ['>=1.22.0', '==1.21.*']
+ python-version: ['3.9', '3.10', '3.11', '3.12']
+ numpy_version: ['>=1.24.0', '==1.23.*']
exclude:
- python-version: '3.10'
- numpy_version: '==1.21.*'
+ numpy_version: '==1.23.*'
- python-version: '3.11'
- numpy_version: '==1.21.*'
+ numpy_version: '==1.23.*'
+ - python-version: '3.12'
+ numpy_version: '==1.23.*'
services:
redis:
image: redis
@@ -42,7 +44,7 @@
with:
fetch-depth: 0
- name: Setup Miniconda
- uses: conda-incubator/setup-miniconda@v3.0.2
+ uses: conda-incubator/setup-miniconda@v3.0.3
with:
channels: conda-forge
python-version: ${{ matrix.python-version }}
@@ -62,7 +64,7 @@
python -m pip install --upgrade pip
python -m pip install -U pip setuptools wheel line_profiler
python -m pip install -rrequirements_dev_minimal.txt
numpy${{matrix.numpy_version}} -rrequirements_dev_optional.txt pymongo redis
- python -m pip install .
+ python -m pip install -e .
python -m pip freeze
- name: Tests
shell: "bash -l {0}"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/.github/workflows/releases.yml
new/zarr-2.17.2/.github/workflows/releases.yml
--- old/zarr-2.17.1/.github/workflows/releases.yml 2024-03-06
20:26:17.000000000 +0100
+++ new/zarr-2.17.2/.github/workflows/releases.yml 2024-04-05
22:55:46.000000000 +0200
@@ -16,7 +16,7 @@
submodules: true
fetch-depth: 0
- - uses: actions/setup-python@v5.0.0
+ - uses: actions/setup-python@v5.1.0
name: Install Python
with:
python-version: '3.9'
@@ -64,7 +64,7 @@
with:
name: releases
path: dist
- - uses: pypa/gh-action-pypi-publish@v1.8.12
+ - uses: pypa/gh-action-pypi-publish@v1.8.14
with:
user: __token__
password: ${{ secrets.pypi_password }}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/.github/workflows/windows-testing.yml
new/zarr-2.17.2/.github/workflows/windows-testing.yml
--- old/zarr-2.17.1/.github/workflows/windows-testing.yml 2024-03-06
20:26:17.000000000 +0100
+++ new/zarr-2.17.2/.github/workflows/windows-testing.yml 2024-04-05
22:55:46.000000000 +0200
@@ -21,7 +21,7 @@
- uses: actions/checkout@v4
with:
fetch-depth: 0
- - uses: conda-incubator/setup-miniconda@v3.0.2
+ - uses: conda-incubator/setup-miniconda@v3.0.3
with:
auto-update-conda: true
python-version: ${{ matrix.python-version }}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/.pre-commit-config.yaml
new/zarr-2.17.2/.pre-commit-config.yaml
--- old/zarr-2.17.1/.pre-commit-config.yaml 2024-03-06 20:26:17.000000000
+0100
+++ new/zarr-2.17.2/.pre-commit-config.yaml 2024-04-05 22:55:46.000000000
+0200
@@ -8,11 +8,11 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
- rev: 'v0.2.2'
+ rev: 'v0.3.5'
hooks:
- id: ruff
- repo: https://github.com/psf/black
- rev: 24.2.0
+ rev: 24.3.0
hooks:
- id: black
- repo: https://github.com/codespell-project/codespell
@@ -24,7 +24,7 @@
hooks:
- id: check-yaml
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: v1.8.0
+ rev: v1.9.0
hooks:
- id: mypy
files: zarr
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/PKG-INFO new/zarr-2.17.2/PKG-INFO
--- old/zarr-2.17.1/PKG-INFO 2024-03-06 20:26:26.934069400 +0100
+++ new/zarr-2.17.2/PKG-INFO 2024-04-05 22:55:54.845134000 +0200
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: zarr
-Version: 2.17.1
+Version: 2.17.2
Summary: An implementation of chunked, compressed, N-dimensional arrays for
Python
Maintainer-email: Alistair Miles <aliman...@googlemail.com>
License: MIT
@@ -25,7 +25,7 @@
Description-Content-Type: text/markdown
License-File: LICENSE.txt
Requires-Dist: asciitree
-Requires-Dist: numpy>=1.21.1
+Requires-Dist: numpy>=1.23
Requires-Dist: fasteners; sys_platform != "emscripten"
Requires-Dist: numcodecs>=0.10.0
Provides-Extra: jupyter
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/docs/installation.rst
new/zarr-2.17.2/docs/installation.rst
--- old/zarr-2.17.1/docs/installation.rst 2024-03-06 20:26:17.000000000
+0100
+++ new/zarr-2.17.2/docs/installation.rst 2024-04-05 22:55:46.000000000
+0200
@@ -6,6 +6,11 @@
appropriate for your operating system and Python distribution. Other
dependencies should be
installed automatically if using one of the installation methods below.
+Note: Zarr has endorsed `Scientific-Python SPEC 0
<https://scientific-python.org/specs/spec-0000/>`_ and now follows the version
support window as outlined below:
+
+- Python: 36 months after initial release
+- Core package dependencies (e.g. NumPy): 24 months after initial release
+
Install Zarr from PyPI::
$ pip install zarr
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/docs/release.rst
new/zarr-2.17.2/docs/release.rst
--- old/zarr-2.17.1/docs/release.rst 2024-03-06 20:26:17.000000000 +0100
+++ new/zarr-2.17.2/docs/release.rst 2024-04-05 22:55:46.000000000 +0200
@@ -13,10 +13,38 @@
# to document your changes. On releases it will be
# re-indented so that it does not show up in the notes.
-.. _unreleased:
+.. _release_2.17.2:
-Unreleased
-----------
+2.17.2
+------
+
+Enhancements
+~~~~~~~~~~~~
+* [v3] Dramatically reduce number of ``__contains__`` requests in favor of
optimistically calling `__getitem__`
+ and handling any error that may arise.
+ By :user:`Deepak Cherian <dcherian>`.
+
+* [v3] Reuse the downloaded array metadata when creating an ``Array``.
+ By :user:`Deepak Cherian <dcherian>`.
+
+* Optimize ``Array.info`` so that it calls `getsize` only once.
+ By :user:`Deepak Cherian <dcherian>`.
+
+* Override IPython ``_repr_*_`` methods to avoid expensive lookups against
object stores.
+ By :user:`Deepak Cherian <dcherian>` :issue:`1716`.
+
+* FSStore now raises rather than return bad data.
+ By :user:`Martin Durant <martindurant>` and :user:`Ian Carroll <itcarroll>`
:issue:`1604`.
+
+Maintenance
+~~~~~~~~~~~
+
+* Add CI test environment for Python 3.12
+ By :user:`Joe Hamman <jhamman>` :issue:`1719`.
+
+* Bump minimum supported NumPy version to 1.23 (per spec 0000)
+ By :user:`Joe Hamman <jhamman>` :issue:`1719`.
+
.. _release_2.17.1:
@@ -1610,11 +1638,11 @@
Documentation
~~~~~~~~~~~~~
-* Some changes have been made to the :ref:`spec_v2` document to clarify
+* Some changes have been made to the Zarr Specification v2 document to clarify
ambiguities and add some missing information. These changes do not break
compatibility
with any of the material as previously implemented, and so the changes have
been made
in-place in the document without incrementing the document version number.
See the
- section on :ref:`spec_v2_changes` in the specification document for more
information.
+ section on changes in the specification document for more information.
* A new :ref:`tutorial_indexing` section has been added to the tutorial.
* A new :ref:`tutorial_strings` section has been added to the tutorial
(:issue:`135`, :issue:`175`).
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/docs/spec/v1.rst
new/zarr-2.17.2/docs/spec/v1.rst
--- old/zarr-2.17.1/docs/spec/v1.rst 2024-03-06 20:26:17.000000000 +0100
+++ new/zarr-2.17.2/docs/spec/v1.rst 2024-04-05 22:55:46.000000000 +0200
@@ -3,268 +3,5 @@
Zarr Storage Specification Version 1
====================================
-This document provides a technical specification of the protocol and
-format used for storing a Zarr array. The key words "MUST", "MUST
-NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT",
-"RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be
-interpreted as described in `RFC 2119
-<https://www.ietf.org/rfc/rfc2119.txt>`_.
-
-Status
-------
-
-This specification is deprecated. See :ref:`spec` for the latest version.
-
-Storage
--------
-
-A Zarr array can be stored in any storage system that provides a
-key/value interface, where a key is an ASCII string and a value is an
-arbitrary sequence of bytes, and the supported operations are read
-(get the sequence of bytes associated with a given key), write (set
-the sequence of bytes associated with a given key) and delete (remove
-a key/value pair).
-
-For example, a directory in a file system can provide this interface,
-where keys are file names, values are file contents, and files can be
-read, written or deleted via the operating system. Equally, an S3
-bucket can provide this interface, where keys are resource names,
-values are resource contents, and resources can be read, written or
-deleted via HTTP.
-
-Below an "array store" refers to any system implementing this
-interface.
-
-Metadata
---------
-
-Each array requires essential configuration metadata to be stored,
-enabling correct interpretation of the stored data. This metadata is
-encoded using JSON and stored as the value of the 'meta' key within an
-array store.
-
-The metadata resource is a JSON object. The following keys MUST be
-present within the object:
-
-zarr_format
- An integer defining the version of the storage specification to which the
- array store adheres.
-shape
- A list of integers defining the length of each dimension of the array.
-chunks
- A list of integers defining the length of each dimension of a chunk of the
- array. Note that all chunks within a Zarr array have the same shape.
-dtype
- A string or list defining a valid data type for the array. See also
- the subsection below on data type encoding.
-compression
- A string identifying the primary compression library used to compress
- each chunk of the array.
-compression_opts
- An integer, string or dictionary providing options to the primary
- compression library.
-fill_value
- A scalar value providing the default value to use for uninitialized
- portions of the array.
-order
- Either 'C' or 'F', defining the layout of bytes within each chunk of the
- array. 'C' means row-major order, i.e., the last dimension varies fastest;
- 'F' means column-major order, i.e., the first dimension varies fastest.
-
-Other keys MAY be present within the metadata object however they MUST
-NOT alter the interpretation of the required fields defined above.
-
-For example, the JSON object below defines a 2-dimensional array of
-64-bit little-endian floating point numbers with 10000 rows and 10000
-columns, divided into chunks of 1000 rows and 1000 columns (so there
-will be 100 chunks in total arranged in a 10 by 10 grid). Within each
-chunk the data are laid out in C contiguous order, and each chunk is
-compressed using the Blosc compression library::
-
- {
- "chunks": [
- 1000,
- 1000
- ],
- "compression": "blosc",
- "compression_opts": {
- "clevel": 5,
- "cname": "lz4",
- "shuffle": 1
- },
- "dtype": "<f8",
- "fill_value": null,
- "order": "C",
- "shape": [
- 10000,
- 10000
- ],
- "zarr_format": 1
- }
-
-Data type encoding
-~~~~~~~~~~~~~~~~~~
-
-Simple data types are encoded within the array metadata resource as a
-string, following the `NumPy array protocol type string (typestr)
-format
-<numpy:arrays.interface>`_. The
-format consists of 3 parts: a character describing the byteorder of
-the data (``<``: little-endian, ``>``: big-endian, ``|``:
-not-relevant), a character code giving the basic type of the array,
-and an integer providing the number of bytes the type uses. The byte
-order MUST be specified. E.g., ``"<f8"``, ``">i4"``, ``"|b1"`` and
-``"|S12"`` are valid data types.
-
-Structure data types (i.e., with multiple named fields) are encoded as
-a list of two-element lists, following `NumPy array protocol type
-descriptions (descr)
-<numpy:arrays.interface>`_.
-For example, the JSON list ``[["r", "|u1"], ["g", "|u1"], ["b",
-"|u1"]]`` defines a data type composed of three single-byte unsigned
-integers labelled 'r', 'g' and 'b'.
-
-Chunks
-------
-
-Each chunk of the array is compressed by passing the raw bytes for the
-chunk through the primary compression library to obtain a new sequence
-of bytes comprising the compressed chunk data. No header is added to
-the compressed bytes or any other modification made. The internal
-structure of the compressed bytes will depend on which primary
-compressor was used. For example, the `Blosc compressor
-<https://github.com/Blosc/c-blosc/blob/main/README_HEADER.rst>`_
-produces a sequence of bytes that begins with a 16-byte header
-followed by compressed data.
-
-The compressed sequence of bytes for each chunk is stored under a key
-formed from the index of the chunk within the grid of chunks
-representing the array. To form a string key for a chunk, the indices
-are converted to strings and concatenated with the period character
-('.') separating each index. For example, given an array with shape
-(10000, 10000) and chunk shape (1000, 1000) there will be 100 chunks
-laid out in a 10 by 10 grid. The chunk with indices (0, 0) provides
-data for rows 0-999 and columns 0-999 and is stored under the key
-'0.0'; the chunk with indices (2, 4) provides data for rows 2000-2999
-and columns 4000-4999 and is stored under the key '2.4'; etc.
-
-There is no need for all chunks to be present within an array
-store. If a chunk is not present then it is considered to be in an
-uninitialized state. An uninitialized chunk MUST be treated as if it
-was uniformly filled with the value of the 'fill_value' field in the
-array metadata. If the 'fill_value' field is ``null`` then the
-contents of the chunk are undefined.
-
-Note that all chunks in an array have the same shape. If the length of
-any array dimension is not exactly divisible by the length of the
-corresponding chunk dimension then some chunks will overhang the edge
-of the array. The contents of any chunk region falling outside the
-array are undefined.
-
-Attributes
-----------
-
-Each array can also be associated with custom attributes, which are
-simple key/value items with application-specific meaning. Custom
-attributes are encoded as a JSON object and stored under the 'attrs'
-key within an array store. Even if the attributes are empty, the
-'attrs' key MUST be present within an array store.
-
-For example, the JSON object below encodes three attributes named
-'foo', 'bar' and 'baz'::
-
- {
- "foo": 42,
- "bar": "apples",
- "baz": [1, 2, 3, 4]
- }
-
-Example
--------
-
-Below is an example of storing a Zarr array, using a directory on the
-local file system as storage.
-
-Initialize the store::
-
- >>> import zarr
- >>> store = zarr.DirectoryStore('example.zarr')
- >>> zarr.init_store(store, shape=(20, 20), chunks=(10, 10),
- ... dtype='i4', fill_value=42, compression='zlib',
- ... compression_opts=1, overwrite=True)
-
-No chunks are initialized yet, so only the 'meta' and 'attrs' keys
-have been set::
-
- >>> import os
- >>> sorted(os.listdir('example.zarr'))
- ['attrs', 'meta']
-
-Inspect the array metadata::
-
- >>> print(open('example.zarr/meta').read())
- {
- "chunks": [
- 10,
- 10
- ],
- "compression": "zlib",
- "compression_opts": 1,
- "dtype": "<i4",
- "fill_value": 42,
- "order": "C",
- "shape": [
- 20,
- 20
- ],
- "zarr_format": 1
- }
-
-Inspect the array attributes::
-
- >>> print(open('example.zarr/attrs').read())
- {}
-
-Set some data::
-
- >>> z = zarr.Array(store)
- >>> z[0:10, 0:10] = 1
- >>> sorted(os.listdir('example.zarr'))
- ['0.0', 'attrs', 'meta']
-
-Set some more data::
-
- >>> z[0:10, 10:20] = 2
- >>> z[10:20, :] = 3
- >>> sorted(os.listdir('example.zarr'))
- ['0.0', '0.1', '1.0', '1.1', 'attrs', 'meta']
-
-Manually decompress a single chunk for illustration::
-
- >>> import zlib
- >>> b = zlib.decompress(open('example.zarr/0.0', 'rb').read())
- >>> import numpy as np
- >>> a = np.frombuffer(b, dtype='<i4')
- >>> a
- array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32)
-
-Modify the array attributes::
-
- >>> z.attrs['foo'] = 42
- >>> z.attrs['bar'] = 'apples'
- >>> z.attrs['baz'] = [1, 2, 3, 4]
- >>> print(open('example.zarr/attrs').read())
- {
- "bar": "apples",
- "baz": [
- 1,
- 2,
- 3,
- 4
- ],
- "foo": 42
- }
+The V1 Specification has been migrated to its website â
+https://zarr-specs.readthedocs.io/.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/docs/spec/v2.rst
new/zarr-2.17.2/docs/spec/v2.rst
--- old/zarr-2.17.1/docs/spec/v2.rst 2024-03-06 20:26:17.000000000 +0100
+++ new/zarr-2.17.2/docs/spec/v2.rst 2024-04-05 22:55:46.000000000 +0200
@@ -3,563 +3,5 @@
Zarr Storage Specification Version 2
====================================
-This document provides a technical specification of the protocol and format
-used for storing Zarr arrays. The key words "MUST", "MUST NOT", "REQUIRED",
-"SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and
-"OPTIONAL" in this document are to be interpreted as described in `RFC 2119
-<https://www.ietf.org/rfc/rfc2119.txt>`_.
-
-Status
-------
-
-This specification is the latest version. See :ref:`spec` for previous
-versions.
-
-.. _spec_v2_storage:
-
-Storage
--------
-
-A Zarr array can be stored in any storage system that provides a key/value
-interface, where a key is an ASCII string and a value is an arbitrary sequence
-of bytes, and the supported operations are read (get the sequence of bytes
-associated with a given key), write (set the sequence of bytes associated with
-a given key) and delete (remove a key/value pair).
-
-For example, a directory in a file system can provide this interface, where
-keys are file names, values are file contents, and files can be read, written
-or deleted via the operating system. Equally, an S3 bucket can provide this
-interface, where keys are resource names, values are resource contents, and
-resources can be read, written or deleted via HTTP.
-
-Below an "array store" refers to any system implementing this interface.
-
-.. _spec_v2_array:
-
-Arrays
-------
-
-.. _spec_v2_array_metadata:
-
-Metadata
-~~~~~~~~
-
-Each array requires essential configuration metadata to be stored, enabling
-correct interpretation of the stored data. This metadata is encoded using JSON
-and stored as the value of the ".zarray" key within an array store.
-
-The metadata resource is a JSON object. The following keys MUST be present
-within the object:
-
-zarr_format
- An integer defining the version of the storage specification to which the
- array store adheres.
-shape
- A list of integers defining the length of each dimension of the array.
-chunks
- A list of integers defining the length of each dimension of a chunk of the
- array. Note that all chunks within a Zarr array have the same shape.
-dtype
- A string or list defining a valid data type for the array. See also
- the subsection below on data type encoding.
-compressor
- A JSON object identifying the primary compression codec and providing
- configuration parameters, or ``null`` if no compressor is to be used.
- The object MUST contain an ``"id"`` key identifying the codec to be used.
-fill_value
- A scalar value providing the default value to use for uninitialized
- portions of the array, or ``null`` if no fill_value is to be used.
-order
- Either "C" or "F", defining the layout of bytes within each chunk of the
- array. "C" means row-major order, i.e., the last dimension varies fastest;
- "F" means column-major order, i.e., the first dimension varies fastest.
-filters
- A list of JSON objects providing codec configurations, or ``null`` if no
- filters are to be applied. Each codec configuration object MUST contain a
- ``"id"`` key identifying the codec to be used.
-
-The following keys MAY be present within the object:
-
-dimension_separator
- If present, either the string ``"."`` or ``"/"`` defining the separator
placed
- between the dimensions of a chunk. If the value is not set, then the
- default MUST be assumed to be ``"."``, leading to chunk keys of the form
"0.0".
- Arrays defined with ``"/"`` as the dimension separator can be considered
to have
- nested, or hierarchical, keys of the form "0/0" that SHOULD where possible
- produce a directory-like structure.
-
-Other keys SHOULD NOT be present within the metadata object and SHOULD be
-ignored by implementations.
-
-For example, the JSON object below defines a 2-dimensional array of 64-bit
-little-endian floating point numbers with 10000 rows and 10000 columns, divided
-into chunks of 1000 rows and 1000 columns (so there will be 100 chunks in total
-arranged in a 10 by 10 grid). Within each chunk the data are laid out in C
-contiguous order. Each chunk is encoded using a delta filter and compressed
-using the Blosc compression library prior to storage::
-
- {
- "chunks": [
- 1000,
- 1000
- ],
- "compressor": {
- "id": "blosc",
- "cname": "lz4",
- "clevel": 5,
- "shuffle": 1
- },
- "dtype": "<f8",
- "fill_value": "NaN",
- "filters": [
- {"id": "delta", "dtype": "<f8", "astype": "<f4"}
- ],
- "order": "C",
- "shape": [
- 10000,
- 10000
- ],
- "zarr_format": 2
- }
-
-.. _spec_v2_array_dtype:
-
-Data type encoding
-~~~~~~~~~~~~~~~~~~
-
-Simple data types are encoded within the array metadata as a string,
-following the :ref:`NumPy array protocol type string (typestr) format
-<numpy:arrays.interface>`. The format
-consists of 3 parts:
-
-* One character describing the byteorder of the data (``"<"``: little-endian;
- ``">"``: big-endian; ``"|"``: not-relevant)
-* One character code giving the basic type of the array (``"b"``: Boolean
(integer
- type where all values are only True or False); ``"i"``: integer; ``"u"``:
unsigned
- integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"m"``:
timedelta;
- ``"M"``: datetime; ``"S"``: string (fixed-length sequence of char); ``"U"``:
unicode
- (fixed-length sequence of Py_UNICODE); ``"V"``: other (void * â each item
is a
- fixed-size chunk of memory))
-* An integer specifying the number of bytes the type uses.
-
-The byte order MUST be specified. E.g., ``"<f8"``, ``">i4"``, ``"|b1"`` and
-``"|S12"`` are valid data type encodings.
-
-For datetime64 ("M") and timedelta64 ("m") data types, these MUST also include
the
-units within square brackets. A list of valid units and their definitions are
given in
-the :ref:`NumPy documentation on Datetimes and Timedeltas
-<numpy:arrays.dtypes.dateunits>`.
-For example, ``"<M8[ns]"`` specifies a datetime64 data type with nanosecond
time units.
-
-Structured data types (i.e., with multiple named fields) are encoded
-as a list of lists, following :ref:`NumPy array protocol type descriptions
-(descr)
-<numpy:arrays.interface>`. Each
-sub-list has the form ``[fieldname, datatype, shape]`` where ``shape``
-is optional. ``fieldname`` is a string, ``datatype`` is a string
-specifying a simple data type (see above), and ``shape`` is a list of
-integers specifying subarray shape. For example, the JSON list below
-defines a data type composed of three single-byte unsigned integer
-fields named "r", "g" and "b"::
-
- [["r", "|u1"], ["g", "|u1"], ["b", "|u1"]]
-
-For example, the JSON list below defines a data type composed of three
-fields named "x", "y" and "z", where "x" and "y" each contain 32-bit
-floats, and each item in "z" is a 2 by 2 array of floats::
-
- [["x", "<f4"], ["y", "<f4"], ["z", "<f4", [2, 2]]]
-
-Structured data types may also be nested, e.g., the following JSON
-list defines a data type with two fields "foo" and "bar", where "bar"
-has two sub-fields "baz" and "qux"::
-
- [["foo", "<f4"], ["bar", [["baz", "<f4"], ["qux", "<i4"]]]]
-
-.. _spec_v2_array_fill_value:
-
-Fill value encoding
-~~~~~~~~~~~~~~~~~~~
-
-For simple floating point data types, the following table MUST be used to
-encode values of the "fill_value" field:
-
-================= ===============
-Value JSON encoding
-================= ===============
-Not a Number ``"NaN"``
-Positive Infinity ``"Infinity"``
-Negative Infinity ``"-Infinity"``
-================= ===============
-
-If an array has a fixed length byte string data type (e.g., ``"|S12"``), or a
-structured data type, and if the fill value is not null, then the fill value
-MUST be encoded as an ASCII string using the standard Base64 alphabet.
-
-.. _spec_v2_array_chunks:
-
-Chunks
-~~~~~~
-
-Each chunk of the array is compressed by passing the raw bytes for the chunk
-through the primary compression library to obtain a new sequence of bytes
-comprising the compressed chunk data. No header is added to the compressed
-bytes or any other modification made. The internal structure of the compressed
-bytes will depend on which primary compressor was used. For example, the `Blosc
-compressor
<https://github.com/Blosc/c-blosc/blob/main/README_CHUNK_FORMAT.rst>`_
-produces a sequence of bytes that begins with a 16-byte header followed by
-compressed data.
-
-The compressed sequence of bytes for each chunk is stored under a key formed
-from the index of the chunk within the grid of chunks representing the array.
-To form a string key for a chunk, the indices are converted to strings and
-concatenated with the period character (".") separating each index. For
-example, given an array with shape (10000, 10000) and chunk shape (1000, 1000)
-there will be 100 chunks laid out in a 10 by 10 grid. The chunk with indices
-(0, 0) provides data for rows 0-999 and columns 0-999 and is stored under the
-key "0.0"; the chunk with indices (2, 4) provides data for rows 2000-2999 and
-columns 4000-4999 and is stored under the key "2.4"; etc.
-
-There is no need for all chunks to be present within an array store. If a chunk
-is not present then it is considered to be in an uninitialized state. An
-uninitialized chunk MUST be treated as if it was uniformly filled with the
value
-of the "fill_value" field in the array metadata. If the "fill_value" field is
-``null`` then the contents of the chunk are undefined.
-
-Note that all chunks in an array have the same shape. If the length of any
-array dimension is not exactly divisible by the length of the corresponding
-chunk dimension then some chunks will overhang the edge of the array. The
-contents of any chunk region falling outside the array are undefined.
-
-.. _spec_v2_array_filters:
-
-Filters
-~~~~~~~
-
-Optionally a sequence of one or more filters can be used to transform chunk
-data prior to compression. When storing data, filters are applied in the order
-specified in array metadata to encode data, then the encoded data are passed to
-the primary compressor. When retrieving data, stored chunk data are
-decompressed by the primary compressor then decoded using filters in the
-reverse order.
-
-.. _spec_v2_hierarchy:
-
-Hierarchies
------------
-
-.. _spec_v2_hierarchy_paths:
-
-Logical storage paths
-~~~~~~~~~~~~~~~~~~~~~
-
-Multiple arrays can be stored in the same array store by associating each array
-with a different logical path. A logical path is simply an ASCII string. The
-logical path is used to form a prefix for keys used by the array. For example,
-if an array is stored at logical path "foo/bar" then the array metadata will be
-stored under the key "foo/bar/.zarray", the user-defined attributes will be
-stored under the key "foo/bar/.zattrs", and the chunks will be stored under
-keys like "foo/bar/0.0", "foo/bar/0.1", etc.
-
-To ensure consistent behaviour across different storage systems, logical paths
-MUST be normalized as follows:
-
-* Replace all backward slash characters ("\\\\") with forward slash characters
- ("/")
-* Strip any leading "/" characters
-* Strip any trailing "/" characters
-* Collapse any sequence of more than one "/" character into a single "/"
- character
-
-The key prefix is then obtained by appending a single "/" character to the
-normalized logical path.
-
-After normalization, if splitting a logical path by the "/" character results
-in any path segment equal to the string "." or the string ".." then an error
-MUST be raised.
-
-N.B., how the underlying array store processes requests to store values under
-keys containing the "/" character is entirely up to the store implementation
-and is not constrained by this specification. E.g., an array store could simply
-treat all keys as opaque ASCII strings; equally, an array store could map
-logical paths onto some kind of hierarchical storage (e.g., directories on a
-file system).
-
-.. _spec_v2_hierarchy_groups:
-
-Groups
-~~~~~~
-
-Arrays can be organized into groups which can also contain other groups. A
-group is created by storing group metadata under the ".zgroup" key under some
-logical path. E.g., a group exists at the root of an array store if the
-".zgroup" key exists in the store, and a group exists at logical path "foo/bar"
-if the "foo/bar/.zgroup" key exists in the store.
-
-If the user requests a group to be created under some logical path, then groups
-MUST also be created at all ancestor paths. E.g., if the user requests group
-creation at path "foo/bar" then groups MUST be created at path "foo" and the
-root of the store, if they don't already exist.
-
-If the user requests an array to be created under some logical path, then
-groups MUST also be created at all ancestor paths. E.g., if the user requests
-array creation at path "foo/bar/baz" then groups must be created at path
-"foo/bar", path "foo", and the root of the store, if they don't already exist.
-
-The group metadata resource is a JSON object. The following keys MUST be
present
-within the object:
-
-zarr_format
- An integer defining the version of the storage specification to which the
- array store adheres.
-
-Other keys MUST NOT be present within the metadata object.
-
-The members of a group are arrays and groups stored under logical paths that
-are direct children of the parent group's logical path. E.g., if groups exist
-under the logical paths "foo" and "foo/bar" and an array exists at logical path
-"foo/baz" then the members of the group at path "foo" are the group at path
-"foo/bar" and the array at path "foo/baz".
-
-.. _spec_v2_attrs:
-
-Attributes
-----------
-
-An array or group can be associated with custom attributes, which are arbitrary
-key/value pairs with application-specific meaning. Custom attributes are
encoded
-as a JSON object and stored under the ".zattrs" key within an array store. The
-".zattrs" key does not have to be present, and if it is absent the attributes
-should be treated as empty.
-
-For example, the JSON object below encodes three attributes named
-"foo", "bar" and "baz"::
-
- {
- "foo": 42,
- "bar": "apples",
- "baz": [1, 2, 3, 4]
- }
-
-.. _spec_v2_examples:
-
-Examples
---------
-
-Storing a single array
-~~~~~~~~~~~~~~~~~~~~~~
-
-Below is an example of storing a Zarr array, using a directory on the
-local file system as storage.
-
-Create an array::
-
- >>> import zarr
- >>> store = zarr.DirectoryStore('data/example.zarr')
- >>> a = zarr.create(shape=(20, 20), chunks=(10, 10), dtype='i4',
- ... fill_value=42, compressor=zarr.Zlib(level=1),
- ... store=store, overwrite=True)
-
-No chunks are initialized yet, so only the ".zarray" and ".zattrs" keys
-have been set in the store::
-
- >>> import os
- >>> sorted(os.listdir('data/example.zarr'))
- ['.zarray']
-
-Inspect the array metadata::
-
- >>> print(open('data/example.zarr/.zarray').read())
- {
- "chunks": [
- 10,
- 10
- ],
- "compressor": {
- "id": "zlib",
- "level": 1
- },
- "dtype": "<i4",
- "fill_value": 42,
- "filters": null,
- "order": "C",
- "shape": [
- 20,
- 20
- ],
- "zarr_format": 2
- }
-
-Chunks are initialized on demand. E.g., set some data::
-
- >>> a[0:10, 0:10] = 1
- >>> sorted(os.listdir('data/example.zarr'))
- ['.zarray', '0.0']
-
-Set some more data::
-
- >>> a[0:10, 10:20] = 2
- >>> a[10:20, :] = 3
- >>> sorted(os.listdir('data/example.zarr'))
- ['.zarray', '0.0', '0.1', '1.0', '1.1']
-
-Manually decompress a single chunk for illustration::
-
- >>> import zlib
- >>> buf = zlib.decompress(open('data/example.zarr/0.0', 'rb').read())
- >>> import numpy as np
- >>> chunk = np.frombuffer(buf, dtype='<i4')
- >>> chunk
- array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32)
-
-Modify the array attributes::
-
- >>> a.attrs['foo'] = 42
- >>> a.attrs['bar'] = 'apples'
- >>> a.attrs['baz'] = [1, 2, 3, 4]
- >>> sorted(os.listdir('data/example.zarr'))
- ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1']
- >>> print(open('data/example.zarr/.zattrs').read())
- {
- "bar": "apples",
- "baz": [
- 1,
- 2,
- 3,
- 4
- ],
- "foo": 42
- }
-
-Storing multiple arrays in a hierarchy
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Below is an example of storing multiple Zarr arrays organized into a group
-hierarchy, using a directory on the local file system as storage. This storage
-implementation maps logical paths onto directory paths on the file system,
-however this is an implementation choice and is not required.
-
-Setup the store::
-
- >>> import zarr
- >>> store = zarr.DirectoryStore('data/group.zarr')
-
-Create the root group::
-
- >>> root_grp = zarr.group(store, overwrite=True)
-
-The metadata resource for the root group has been created::
-
- >>> import os
- >>> sorted(os.listdir('data/group.zarr'))
- ['.zgroup']
-
-Inspect the group metadata::
-
- >>> print(open('data/group.zarr/.zgroup').read())
- {
- "zarr_format": 2
- }
-
-Create a sub-group::
-
- >>> sub_grp = root_grp.create_group('foo')
-
-What has been stored::
-
- >>> sorted(os.listdir('data/group.zarr'))
- ['.zgroup', 'foo']
- >>> sorted(os.listdir('data/group.zarr/foo'))
- ['.zgroup']
-
-Create an array within the sub-group::
-
- >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10))
- >>> a[:] = 42
-
-Set a custom attributes::
-
- >>> a.attrs['comment'] = 'answer to life, the universe and everything'
-
-What has been stored::
-
- >>> sorted(os.listdir('data/group.zarr'))
- ['.zgroup', 'foo']
- >>> sorted(os.listdir('data/group.zarr/foo'))
- ['.zgroup', 'bar']
- >>> sorted(os.listdir('data/group.zarr/foo/bar'))
- ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1']
-
-Here is the same example using a Zip file as storage::
-
- >>> store = zarr.ZipStore('data/group.zip', mode='w')
- >>> root_grp = zarr.group(store)
- >>> sub_grp = root_grp.create_group('foo')
- >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10))
- >>> a[:] = 42
- >>> a.attrs['comment'] = 'answer to life, the universe and everything'
- >>> store.close()
-
-What has been stored::
-
- >>> import zipfile
- >>> zf = zipfile.ZipFile('data/group.zip', mode='r')
- >>> for name in sorted(zf.namelist()):
- ... print(name)
- .zgroup
- foo/.zgroup
- foo/bar/.zarray
- foo/bar/.zattrs
- foo/bar/0.0
- foo/bar/0.1
- foo/bar/1.0
- foo/bar/1.1
-
-.. _spec_v2_changes:
-
-Changes
--------
-
-Version 2 clarifications
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-The following changes have been made to the version 2 specification since it
was
-initially published to clarify ambiguities and add some missing information.
-
-* The specification now describes how bytes fill values should be encoded and
- decoded for arrays with a fixed-length byte string data type (:issue:`165`,
- :issue:`176`).
-
-* The specification now clarifies that units must be specified for datetime64
and
- timedelta64 data types (:issue:`85`, :issue:`215`).
-
-* The specification now clarifies that the '.zattrs' key does not have to be
present for
- either arrays or groups, and if absent then custom attributes should be
treated as
- empty.
-
-* The specification now describes how structured datatypes with
- subarray shapes and/or with nested structured data types are encoded
- in array metadata (:issue:`111`, :issue:`296`).
-
-* Clarified the key/value pairs of custom attributes as "arbitrary" rather than
- "simple".
-
-Changes from version 1 to version 2
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The following changes were made between version 1 and version 2 of this
specification:
-
-* Added support for storing multiple arrays in the same store and organising
- arrays into hierarchies using groups.
-* Array metadata is now stored under the ".zarray" key instead of the "meta"
- key.
-* Custom attributes are now stored under the ".zattrs" key instead of the
- "attrs" key.
-* Added support for filters.
-* Changed encoding of "fill_value" field within array metadata.
-* Changed encoding of compressor information within array metadata to be
- consistent with representation of filter information.
+The V2 Specification has been migrated to its website â
+https://zarr-specs.readthedocs.io/.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/docs/spec/v3.rst
new/zarr-2.17.2/docs/spec/v3.rst
--- old/zarr-2.17.1/docs/spec/v3.rst 2024-03-06 20:26:17.000000000 +0100
+++ new/zarr-2.17.2/docs/spec/v3.rst 2024-04-05 22:55:46.000000000 +0200
@@ -1,7 +1,7 @@
.. _spec_v3:
Zarr Storage Specification Version 3
-=======================================================
+====================================
The V3 Specification has been migrated to its website â
https://zarr-specs.readthedocs.io/.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/docs/tutorial.rst
new/zarr-2.17.2/docs/tutorial.rst
--- old/zarr-2.17.1/docs/tutorial.rst 2024-03-06 20:26:17.000000000 +0100
+++ new/zarr-2.17.2/docs/tutorial.rst 2024-04-05 22:55:46.000000000 +0200
@@ -868,7 +868,7 @@
Distributed File System (`HDFSMap
<https://hdfs3.readthedocs.io/en/latest/api.html#hdfs3.mapping.HDFSMap>`_) and
Google Cloud Storage (`GCSMap
-<http://gcsfs.readthedocs.io/en/latest/api.html#gcsfs.mapping.GCSMap>`_), which
+<https://gcsfs.readthedocs.io/en/latest/api.html#gcsfs.core.GCSFileSystem.get_mapper>`_),
which
can be used with Zarr.
Here is an example using S3Map to read an array created previously::
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/pyproject.toml
new/zarr-2.17.2/pyproject.toml
--- old/zarr-2.17.1/pyproject.toml 2024-03-06 20:26:18.000000000 +0100
+++ new/zarr-2.17.2/pyproject.toml 2024-04-05 22:55:46.000000000 +0200
@@ -13,7 +13,7 @@
requires-python = ">=3.9"
dependencies = [
'asciitree',
- 'numpy>=1.21.1',
+ 'numpy>=1.23',
'fasteners; sys_platform != "emscripten"',
'numcodecs>=0.10.0',
]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/requirements_dev_minimal.txt
new/zarr-2.17.2/requirements_dev_minimal.txt
--- old/zarr-2.17.1/requirements_dev_minimal.txt 2024-03-06
20:26:18.000000000 +0100
+++ new/zarr-2.17.2/requirements_dev_minimal.txt 2024-04-05
22:55:46.000000000 +0200
@@ -5,4 +5,4 @@
msgpack-python==0.5.6
setuptools-scm==8.0.4
# test requirements
-pytest==7.4.4
+pytest==8.1.1
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/requirements_dev_optional.txt
new/zarr-2.17.2/requirements_dev_optional.txt
--- old/zarr-2.17.1/requirements_dev_optional.txt 2024-03-06
20:26:18.000000000 +0100
+++ new/zarr-2.17.2/requirements_dev_optional.txt 2024-04-05
22:55:46.000000000 +0200
@@ -8,15 +8,15 @@
# don't let pyup change pinning for azure-storage-blob, need to pin to older
# version to get compatibility with azure storage emulator on appveyor (FIXME)
azure-storage-blob==12.16.0 # pyup: ignore
-redis==5.0.2
+redis==5.0.3
types-redis
types-setuptools
-pymongo==4.6.2
+pymongo==4.6.3
# optional test requirements
coverage
-pytest-cov==4.1.0
-pytest-doctestplus==1.2.0
-pytest-timeout==2.2.0
+pytest-cov==5.0.0
+pytest-doctestplus==1.2.1
+pytest-timeout==2.3.1
h5py==3.10.0
fsspec==2023.12.2
s3fs==2023.12.2
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr/_storage/store.py
new/zarr-2.17.2/zarr/_storage/store.py
--- old/zarr-2.17.1/zarr/_storage/store.py 2024-03-06 20:26:18.000000000
+0100
+++ new/zarr-2.17.2/zarr/_storage/store.py 2024-04-05 22:55:46.000000000
+0200
@@ -462,7 +462,7 @@
def __eq__(self, other):
return (
- type(self) == type(other)
+ type(self) is type(other)
and self._inner_store == other._inner_store
and self.get_config() == other.get_config()
)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr/attrs.py
new/zarr-2.17.2/zarr/attrs.py
--- old/zarr-2.17.1/zarr/attrs.py 2024-03-06 20:26:18.000000000 +0100
+++ new/zarr-2.17.2/zarr/attrs.py 2024-04-05 22:55:46.000000000 +0200
@@ -25,14 +25,16 @@
"""
- def __init__(self, store, key=".zattrs", read_only=False, cache=True,
synchronizer=None):
+ def __init__(
+ self, store, key=".zattrs", read_only=False, cache=True,
synchronizer=None, cached_dict=None
+ ):
self._version = getattr(store, "_store_version", 2)
_Store = Store if self._version == 2 else StoreV3
self.store = _Store._ensure_store(store)
self.key = key
self.read_only = read_only
self.cache = cache
- self._cached_asdict = None
+ self._cached_asdict = cached_dict if cache else None
self.synchronizer = synchronizer
def _get_nosync(self):
@@ -149,19 +151,20 @@
if self.cache:
self._cached_asdict = d
else:
- if self.key in self.store:
+ try:
+ meta_unparsed = self.store[self.key]
# Cannot write the attributes directly to JSON, but have to
# store it within the pre-existing attributes key of the v3
# metadata.
# Note: this changes the store.counter result in
test_caching_on!
- meta =
self.store._metadata_class.parse_metadata(self.store[self.key])
+ meta = self.store._metadata_class.parse_metadata(meta_unparsed)
if "attributes" in meta and "filters" in meta["attributes"]:
# need to preserve any existing "filters" attribute
d["attributes"]["filters"] = meta["attributes"]["filters"]
meta["attributes"] = d["attributes"]
- else:
+ except KeyError:
meta = d
self.store[self.key] = json_dumps(meta)
if self.cache:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr/core.py new/zarr-2.17.2/zarr/core.py
--- old/zarr-2.17.1/zarr/core.py 2024-03-06 20:26:18.000000000 +0100
+++ new/zarr-2.17.2/zarr/core.py 2024-04-05 22:55:46.000000000 +0200
@@ -172,11 +172,15 @@
# initialize attributes
akey = _prefix_to_attrs_key(self._store, self._key_prefix)
self._attrs = Attributes(
- store, key=akey, read_only=read_only, synchronizer=synchronizer,
cache=cache_attrs
+ store,
+ key=akey,
+ read_only=read_only,
+ synchronizer=synchronizer,
+ cache=cache_attrs,
+ cached_dict=self._meta["attributes"] if self._version == 3 else
None,
)
# initialize info reporter
- self._info_reporter = InfoReporter(self)
# initialize indexing helpers
self._oindex = OIndex(self)
@@ -2429,7 +2433,7 @@
Chunks initialized : 0/10
"""
- return self._info_reporter
+ return InfoReporter(self)
def info_items(self):
return self._synchronized_op(self._info_items_nosync)
@@ -2471,14 +2475,16 @@
items += [("Synchronizer type", typestr(self._synchronizer))]
# storage info
+ nbytes = self.nbytes
+ nbytes_stored = self.nbytes_stored
items += [("Store type", typestr(self._store))]
if self._chunk_store is not None:
items += [("Chunk store type", typestr(self._chunk_store))]
- items += [("No. bytes", bytestr(self.nbytes))]
- if self.nbytes_stored > 0:
+ items += [("No. bytes", bytestr(nbytes))]
+ if nbytes_stored > 0:
items += [
- ("No. bytes stored", bytestr(self.nbytes_stored)),
- ("Storage ratio", f"{self.nbytes / self.nbytes_stored:.1f}"),
+ ("No. bytes stored", bytestr(nbytes_stored)),
+ ("Storage ratio", f"{nbytes / nbytes_stored:.1f}"),
]
items += [("Chunks initialized",
f"{self.nchunks_initialized}/{self.nchunks}")]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr/hierarchy.py
new/zarr-2.17.2/zarr/hierarchy.py
--- old/zarr-2.17.1/zarr/hierarchy.py 2024-03-06 20:26:18.000000000 +0100
+++ new/zarr-2.17.2/zarr/hierarchy.py 2024-04-05 22:55:46.000000000 +0200
@@ -27,6 +27,7 @@
from zarr.errors import (
ContainsArrayError,
ContainsGroupError,
+ ArrayNotFoundError,
GroupNotFoundError,
ReadOnlyError,
)
@@ -207,11 +208,15 @@
# object can still be created.
akey = mkey
self._attrs = Attributes(
- store, key=akey, read_only=read_only, cache=cache_attrs,
synchronizer=synchronizer
+ store,
+ key=akey,
+ read_only=read_only,
+ cache=cache_attrs,
+ synchronizer=synchronizer,
+ cached_dict=self._meta["attributes"] if self._version == 3 and
self._meta else None,
)
# setup info
- self._info = InfoReporter(self)
@property
def store(self):
@@ -266,7 +271,7 @@
@property
def info(self):
"""Return diagnostic information about the group."""
- return self._info
+ return InfoReporter(self)
@property
def meta_array(self):
@@ -458,7 +463,7 @@
"""
path = self._item_path(item)
- if contains_array(self._store, path):
+ try:
return Array(
self._store,
read_only=self._read_only,
@@ -469,7 +474,10 @@
zarr_version=self._version,
meta_array=self._meta_array,
)
- elif contains_group(self._store, path, explicit_only=True):
+ except ArrayNotFoundError:
+ pass
+
+ try:
return Group(
self._store,
read_only=self._read_only,
@@ -480,7 +488,10 @@
zarr_version=self._version,
meta_array=self._meta_array,
)
- elif self._version == 3:
+ except GroupNotFoundError:
+ pass
+
+ if self._version == 3:
implicit_group = meta_root + path + "/"
# non-empty folder in the metadata path implies an implicit group
if self._store.list_prefix(implicit_group):
@@ -515,6 +526,13 @@
raise KeyError(item)
def __getattr__(self, item):
+ # https://github.com/jupyter/notebook/issues/2014
+ # Save a possibly expensive lookup (for e.g. against cloud stores)
+ # Note: The _ipython_display_ method is required to display the right
info as a side-effect.
+ # It is simpler to pretend it doesn't exist.
+ if item in ["_ipython_canary_method_should_not_exist_",
"_ipython_display_"]:
+ raise AttributeError
+
# allow access to group members via dot notation
try:
return self.__getitem__(item)
@@ -1331,6 +1349,40 @@
self._write_op(self._move_nosync, source, dest)
+ # Override ipython repr methods, GH1716
+ #
https://ipython.readthedocs.io/en/stable/config/integrating.html#custom-methods
+ # " If the methods donât exist, the standard repr() is used. If a
method exists and
+ # returns None, it is treated the same as if it does not exist."
+ def _repr_html_(self):
+ return None
+
+ def _repr_latex_(self):
+ return None
+
+ def _repr_mimebundle_(self, **kwargs):
+ return None
+
+ def _repr_svg_(self):
+ return None
+
+ def _repr_png_(self):
+ return None
+
+ def _repr_jpeg_(self):
+ return None
+
+ def _repr_markdown_(self):
+ return None
+
+ def _repr_javascript_(self):
+ return None
+
+ def _repr_pdf_(self):
+ return None
+
+ def _repr_json_(self):
+ return None
+
def _normalize_store_arg(store, *, storage_options=None, mode="r",
zarr_version=None):
if zarr_version is None:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr/storage.py
new/zarr-2.17.2/zarr/storage.py
--- old/zarr-2.17.1/zarr/storage.py 2024-03-06 20:26:18.000000000 +0100
+++ new/zarr-2.17.2/zarr/storage.py 2024-04-05 22:55:46.000000000 +0200
@@ -1417,11 +1417,23 @@
def getitems(
self, keys: Sequence[str], *, contexts: Mapping[str, Context]
) -> Mapping[str, Any]:
- keys_transformed = [self._normalize_key(key) for key in keys]
- results = self.map.getitems(keys_transformed, on_error="omit")
- # The function calling this method may not recognize the transformed
keys
- # So we send the values returned by self.map.getitems back into the
original key space.
- return {keys[keys_transformed.index(rk)]: rv for rk, rv in
results.items()}
+ keys_transformed = {self._normalize_key(key): key for key in keys}
+ results_transformed = self.map.getitems(list(keys_transformed),
on_error="return")
+ results = {}
+ for k, v in results_transformed.items():
+ if isinstance(v, self.exceptions):
+ # Cause recognized exceptions to prompt a KeyError in the
+ # function calling this method
+ continue
+ elif isinstance(v, Exception):
+ # Raise any other exception
+ raise v
+ else:
+ # The function calling this method may not recognize the
transformed
+ # keys, so we send the values returned by self.map.getitems
back into
+ # the original key space.
+ results[keys_transformed[k]] = v
+ return results
def __getitem__(self, key):
key = self._normalize_key(key)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr/tests/test_core.py
new/zarr-2.17.2/zarr/tests/test_core.py
--- old/zarr-2.17.1/zarr/tests/test_core.py 2024-03-06 20:26:18.000000000
+0100
+++ new/zarr-2.17.2/zarr/tests/test_core.py 2024-04-05 22:55:46.000000000
+0200
@@ -123,7 +123,7 @@
"compressor": kwargs.pop("compressor", self.compressor),
"chunk_store": chunk_store,
"storage_transformers": self.create_storage_transformers(shape),
- "filters": kwargs.pop("filters",
self.create_filters(kwargs.get("dtype", None))),
+ "filters": kwargs.pop("filters",
self.create_filters(kwargs.get("dtype"))),
}
# keyword arguments for array instantiation
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr/tests/test_hierarchy.py
new/zarr-2.17.2/zarr/tests/test_hierarchy.py
--- old/zarr-2.17.1/zarr/tests/test_hierarchy.py 2024-03-06
20:26:18.000000000 +0100
+++ new/zarr-2.17.2/zarr/tests/test_hierarchy.py 2024-04-05
22:55:46.000000000 +0200
@@ -1,4 +1,5 @@
import atexit
+import operator
import os
import sys
import pickle
@@ -87,6 +88,26 @@
)
return g
+ def test_ipython_repr_methods(self):
+ g = self.create_group()
+ for method in [
+ "html",
+ "json",
+ "javascript",
+ "markdown",
+ "svg",
+ "png",
+ "jpeg",
+ "latex",
+ "pdf",
+ "mimebundle",
+ ]:
+ assert operator.methodcaller(f"_repr_{method}_")(g) is None
+ with pytest.raises(AttributeError):
+ g._ipython_display_()
+ with pytest.raises(AttributeError):
+ g._ipython_canary_method_should_not_exist_()
+
def test_group_init_1(self):
store, chunk_store = self.create_store()
g = self.create_group(store, chunk_store=chunk_store)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr/tests/test_storage.py
new/zarr-2.17.2/zarr/tests/test_storage.py
--- old/zarr-2.17.1/zarr/tests/test_storage.py 2024-03-06 20:26:18.000000000
+0100
+++ new/zarr-2.17.2/zarr/tests/test_storage.py 2024-04-05 22:55:46.000000000
+0200
@@ -1098,6 +1098,12 @@
@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec")
class TestFSStore(StoreTests):
+ @pytest.fixture
+ def memory_store(self):
+ store = FSStore("memory://")
+ yield store
+ store.fs.store.clear()
+
def create_store(self, normalize_keys=False, dimension_separator=".",
path=None, **kwargs):
if path is None:
path = tempfile.mkdtemp()
@@ -1337,6 +1343,25 @@
)
assert (a[:] == -np.ones((8, 8, 8))).all()
+ def test_exceptions(self, memory_store):
+ fs = memory_store.fs
+ group = zarr.open(memory_store, mode="w")
+ x = group.create_dataset("x", data=[1, 2, 3])
+ y = group.create_dataset("y", data=1)
+ fs.store["/x/0"] = None
+ fs.store["/y/0"] = None
+ # no exception from FSStore.getitems getting KeyError
+ assert group.store.getitems(["foo"], contexts={}) == {}
+ # exception from FSStore.getitems getting AttributeError
+ with pytest.raises(Exception):
+ group.store.getitems(["x/0"], contexts={})
+ # exception from FSStore.getitems getting AttributeError
+ with pytest.raises(Exception):
+ x[...]
+ # exception from FSStore.__getitem__ getting AttributeError
+ with pytest.raises(Exception):
+ y[...]
+
@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec")
class TestFSStoreWithKeySeparator(StoreTests):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr/util.py new/zarr-2.17.2/zarr/util.py
--- old/zarr-2.17.1/zarr/util.py 2024-03-06 20:26:18.000000000 +0100
+++ new/zarr-2.17.2/zarr/util.py 2024-04-05 22:55:46.000000000 +0200
@@ -408,14 +408,13 @@
class InfoReporter:
def __init__(self, obj):
self.obj = obj
+ self.items = self.obj.info_items()
def __repr__(self):
- items = self.obj.info_items()
- return info_text_report(items)
+ return info_text_report(self.items)
def _repr_html_(self):
- items = self.obj.info_items()
- return info_html_report(items)
+ return info_html_report(self.items)
class TreeNode:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr/version.py
new/zarr-2.17.2/zarr/version.py
--- old/zarr-2.17.1/zarr/version.py 2024-03-06 20:26:24.000000000 +0100
+++ new/zarr-2.17.2/zarr/version.py 2024-04-05 22:55:52.000000000 +0200
@@ -12,5 +12,5 @@
__version_tuple__: VERSION_TUPLE
version_tuple: VERSION_TUPLE
-__version__ = version = '2.17.1'
-__version_tuple__ = version_tuple = (2, 17, 1)
+__version__ = version = '2.17.2'
+__version_tuple__ = version_tuple = (2, 17, 2)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr.egg-info/PKG-INFO
new/zarr-2.17.2/zarr.egg-info/PKG-INFO
--- old/zarr-2.17.1/zarr.egg-info/PKG-INFO 2024-03-06 20:26:24.000000000
+0100
+++ new/zarr-2.17.2/zarr.egg-info/PKG-INFO 2024-04-05 22:55:52.000000000
+0200
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: zarr
-Version: 2.17.1
+Version: 2.17.2
Summary: An implementation of chunked, compressed, N-dimensional arrays for
Python
Maintainer-email: Alistair Miles <aliman...@googlemail.com>
License: MIT
@@ -25,7 +25,7 @@
Description-Content-Type: text/markdown
License-File: LICENSE.txt
Requires-Dist: asciitree
-Requires-Dist: numpy>=1.21.1
+Requires-Dist: numpy>=1.23
Requires-Dist: fasteners; sys_platform != "emscripten"
Requires-Dist: numcodecs>=0.10.0
Provides-Extra: jupyter
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/zarr-2.17.1/zarr.egg-info/requires.txt
new/zarr-2.17.2/zarr.egg-info/requires.txt
--- old/zarr-2.17.1/zarr.egg-info/requires.txt 2024-03-06 20:26:24.000000000
+0100
+++ new/zarr-2.17.2/zarr.egg-info/requires.txt 2024-04-05 22:55:52.000000000
+0200
@@ -1,5 +1,5 @@
asciitree
-numpy>=1.21.1
+numpy>=1.23
numcodecs>=0.10.0
[:sys_platform != "emscripten"]
