Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-gcsfs for openSUSE:Factory checked in at 2023-03-27 18:15:37 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-gcsfs (Old) and /work/SRC/openSUSE:Factory/.python-gcsfs.new.31432 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-gcsfs" Mon Mar 27 18:15:37 2023 rev:15 rq:1074479 version:2023.3.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-gcsfs/python-gcsfs.changes 2022-11-21 16:19:37.447863553 +0100 +++ /work/SRC/openSUSE:Factory/.python-gcsfs.new.31432/python-gcsfs.changes 2023-03-27 18:15:38.366903719 +0200 @@ -1,0 +2,17 @@ +Fri Mar 24 15:48:08 UTC 2023 - Ben Greiner <[email protected]> + +- Update to 2023.3.0 + * Don't let find() mess up dircache (#531) + * Drop py3.7 (#529) + * Update docs (#528) + * Make times UTC (#527) + * Use BytesIO for large bodies (#525) + * Fix: Don't append generation when it is absent (#523) + * get/put/cp consistency tests (#521) +- Release 2023.1.0 + * Support create time (#516, 518) + * defer async session creation (#513, 514) + * support listing of file versions (#509) + * fix sign following versioned split protocol (#513) + +------------------------------------------------------------------- Old: ---- gcsfs-2022.11.0-gh.tar.gz New: ---- gcsfs-2023.3.0-gh.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-gcsfs.spec ++++++ --- /var/tmp/diff_new_pack.mMb08O/_old 2023-03-27 18:15:38.862906336 +0200 +++ /var/tmp/diff_new_pack.mMb08O/_new 2023-03-27 18:15:38.866906357 +0200 @@ -1,7 +1,7 @@ # # spec file for package python-gcsfs # -# Copyright (c) 2022 SUSE LLC +# Copyright (c) 2023 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -17,7 +17,7 @@ Name: python-gcsfs -Version: 2022.11.0 +Version: 2023.3.0 Release: 0 Summary: Filesystem interface over GCS License: BSD-3-Clause @@ -25,7 +25,7 @@ URL: https://github.com/fsspec/gcsfs # Use the GitHub tarball for test data Source: https://github.com/fsspec/gcsfs/archive/refs/tags/%{version}.tar.gz#/gcsfs-%{version}-gh.tar.gz -BuildRequires: %{python_module base >= 3.7} +BuildRequires: %{python_module base >= 3.8} BuildRequires: %{python_module setuptools} BuildRequires: fdupes BuildRequires: python-rpm-macros ++++++ gcsfs-2022.11.0-gh.tar.gz -> gcsfs-2023.3.0-gh.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/.github/workflows/ci.yml new/gcsfs-2023.3.0/.github/workflows/ci.yml --- old/gcsfs-2022.11.0/.github/workflows/ci.yml 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/.github/workflows/ci.yml 2023-03-04 21:33:12.000000000 +0100 @@ -2,49 +2,53 @@ on: [push, pull_request, workflow_dispatch] +defaults: + run: + shell: bash -l -eo pipefail {0} + jobs: test: name: Python ${{ matrix.python-version }} runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - name: Checkout source - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Conda Environment - uses: conda-incubator/setup-miniconda@v2 + uses: mamba-org/provision-with-micromamba@main with: - auto-update-conda: true - miniconda-version: latest - activate-environment: test - python-version: ${{ matrix.python-version }} + cache-downloads: true + environment-file: environment_gcsfs.yaml + environment-name: gcsfs_test + extra-specs: | + python=${{ matrix.python-version }} - - name: Install dependencies - shell: bash -l {0} + - name: Conda info run: | - conda install -c conda-forge pytest ujson requests decorator google-auth aiohttp google-auth-oauthlib google-cloud-core google-api-core google-api-python-client -y - pip install git+https://github.com/fsspec/filesystem_spec --no-deps conda list conda --version - - name: Install - shell: bash -l {0} - run: pip install .[crc] + - name: Install libfuse + run: (sudo apt-get install -y fuse || echo "Error installing fuse.") - - name: Run Tests - shell: bash -l {0} + - name: Run tests run: | - export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json - py.test -vv gcsfs + export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json + pytest -vv \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --log-date-format="%H:%M:%S" \ + gcsfs/ lint: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/[email protected] + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - uses: pre-commit/[email protected] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/.isort.cfg new/gcsfs-2023.3.0/.isort.cfg --- old/gcsfs-2022.11.0/.isort.cfg 1970-01-01 01:00:00.000000000 +0100 +++ new/gcsfs-2023.3.0/.isort.cfg 2023-03-04 21:33:12.000000000 +0100 @@ -0,0 +1,2 @@ +[settings] +known_third_party = aiohttp,click,decorator,fsspec,fuse,google,google_auth_oauthlib,pytest,requests,setuptools diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/.pre-commit-config.yaml new/gcsfs-2023.3.0/.pre-commit-config.yaml --- old/gcsfs-2022.11.0/.pre-commit-config.yaml 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/.pre-commit-config.yaml 2023-03-04 21:33:12.000000000 +0100 @@ -1,16 +1,28 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks +exclude: versioneer.py repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.4.0 hooks: - - id: trailing-whitespace - id: end-of-file-fixer - - repo: https://github.com/ambv/black - rev: 22.3.0 + - id: requirements-txt-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.10.0 hooks: - - id: black - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + - id: black + args: + - --target-version=py37 + - repo: https://github.com/pycqa/flake8 + rev: 6.0.0 hooks: - id: flake8 + - repo: https://github.com/asottile/seed-isort-config + rev: v2.2.0 + hooks: + - id: seed-isort-config + - repo: https://github.com/pre-commit/mirrors-isort + rev: v5.7.0 + hooks: + - id: isort diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/.readthedocs.yaml new/gcsfs-2023.3.0/.readthedocs.yaml --- old/gcsfs-2022.11.0/.readthedocs.yaml 1970-01-01 01:00:00.000000000 +0100 +++ new/gcsfs-2023.3.0/.readthedocs.yaml 2023-03-04 21:33:12.000000000 +0100 @@ -0,0 +1,18 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: miniconda3-4.7 + +conda: + environment: docs/environment.yml + +python: + install: + - method: pip + path: . + +sphinx: + configuration: docs/source/conf.py + fail_on_warning: true diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/LICENSE.txt new/gcsfs-2023.3.0/LICENSE.txt --- old/gcsfs-2022.11.0/LICENSE.txt 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/LICENSE.txt 2023-03-04 21:33:12.000000000 +0100 @@ -1,4 +1,4 @@ -BSD 3-Clause License +BSD 3-Clause License Copyright (c) 2014-2018, Anaconda, Inc. and contributors All rights reserved. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/docs/environment.yml new/gcsfs-2023.3.0/docs/environment.yml --- old/gcsfs-2022.11.0/docs/environment.yml 1970-01-01 01:00:00.000000000 +0100 +++ new/gcsfs-2023.3.0/docs/environment.yml 2023-03-04 21:33:12.000000000 +0100 @@ -0,0 +1,8 @@ +name: s3fs +channels: + - defaults +dependencies: + - python= 3.9 + - docutils<0.17 + - sphinx + - sphinx_rtd_theme diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/docs/requirements.txt new/gcsfs-2023.3.0/docs/requirements.txt --- old/gcsfs-2022.11.0/docs/requirements.txt 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/docs/requirements.txt 1970-01-01 01:00:00.000000000 +0100 @@ -1,2 +0,0 @@ -numpydoc -docutils<0.18 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/docs/source/_static/custom.css new/gcsfs-2023.3.0/docs/source/_static/custom.css --- old/gcsfs-2022.11.0/docs/source/_static/custom.css 1970-01-01 01:00:00.000000000 +0100 +++ new/gcsfs-2023.3.0/docs/source/_static/custom.css 2023-03-04 21:33:12.000000000 +0100 @@ -0,0 +1,5 @@ +.classifier:before { + font-style: normal; + margin: 0.5em; + content: ":"; +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/docs/source/api.rst new/gcsfs-2023.3.0/docs/source/api.rst --- old/gcsfs-2022.11.0/docs/source/api.rst 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/docs/source/api.rst 2023-03-04 21:33:12.000000000 +0100 @@ -38,8 +38,10 @@ .. autoclass:: GCSFileSystem :members: + :inherited-members: .. autoclass:: GCSFile :members: + :inherited-members: .. currentmodule:: gcsfs.mapping diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/docs/source/changelog.rst new/gcsfs-2023.3.0/docs/source/changelog.rst --- old/gcsfs-2022.11.0/docs/source/changelog.rst 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/docs/source/changelog.rst 2023-03-04 21:33:12.000000000 +0100 @@ -1,6 +1,25 @@ Changelog ========= +2023.3.0 +-------- + +* Don't let find() mess up dircache (#531) +* Drop py3.7 (#529) +* Update docs (#528) +* Make times UTC (#527) +* Use BytesIO for large bodies (#525) +* Fix: Don't append generation when it is absent (#523) +* get/put/cp consistency tests (#521) + +2023.1.0 +-------- + +* Support create time (#516, 518) +* defer async session creation (#513, 514) +* support listing of file versions (#509) +* fix ``sign`` following versioned split protocol (#513) + 2022.11.0 --------- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/docs/source/conf.py new/gcsfs-2023.3.0/docs/source/conf.py --- old/gcsfs-2022.11.0/docs/source/conf.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/docs/source/conf.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # GCSFs documentation build configuration file, created by # sphinx-quickstart on Mon Mar 21 15:20:01 2016. @@ -13,9 +12,6 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys -import os - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -36,7 +32,7 @@ "sphinx.ext.viewcode", "sphinx.ext.autosummary", "sphinx.ext.extlinks", - "numpydoc", + "sphinx.ext.napoleon", ] # Add any paths that contain templates here, relative to this directory. @@ -69,13 +65,6 @@ # The full version, including alpha/beta/rc tags. release = version -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' @@ -116,15 +105,7 @@ # -- Options for HTML output ---------------------------------------------- -# Taken from docs.readthedocs.io: -# on_rtd is whether we are on readthedocs.io -on_rtd = os.environ.get("READTHEDOCS", None) == "True" - -if not on_rtd: # only import and set the theme if we're building docs locally - import sphinx_rtd_theme - - html_theme = "sphinx_rtd_theme" - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -155,6 +136,10 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] +# Custom CSS file to override read the docs default CSS. +# Contains workaround for RTD not rendering colon between argument name and type +html_css_files = ["custom.css"] + # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. @@ -298,4 +283,4 @@ # If true, do not generate a @detailmenu in the "Top" node's menu. # texinfo_no_detailmenu = False -extlinks = {"pr": ("https://github.com/fsspec/gcsfs/pull/%s";, "PR #")} +extlinks = {"pr": ("https://github.com/fsspec/gcsfs/pull/%s";, "PR #%s")} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/docs/source/fuse.rst new/gcsfs-2023.3.0/docs/source/fuse.rst --- old/gcsfs-2022.11.0/docs/source/fuse.rst 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/docs/source/fuse.rst 2023-03-04 21:33:12.000000000 +0100 @@ -1,7 +1,7 @@ GCSFS and FUSE ============== -Warning, this functionality is **experimental** +Warning, this functionality is **experimental**. FUSE_ is a mechanism to mount user-level filesystems in unix-like systems (linux, osx, etc.). GCSFS is able to use FUSE to present remote @@ -23,10 +23,10 @@ - fusepy_, which can be installed via conda or pip - pandas, which can also be installed via conda or pip (this library is - used only for its timestring parsing. + used only for its timestring parsing). .. _osxfuse: https://osxfuse.github.io/ -.. _fusepy: https://github.com/terencehonles/fusepy +.. _fusepy: https://github.com/fusepy/fusepy Usage ----- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/docs/source/index.rst new/gcsfs-2023.3.0/docs/source/index.rst --- old/gcsfs-2022.11.0/docs/source/index.rst 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/docs/source/index.rst 2023-03-04 21:33:12.000000000 +0100 @@ -10,7 +10,7 @@ .. _github: https://github.com/fsspec/gcsfs/issues -This package depends on fsspec_ , and inherits many useful behaviours from there, +This package depends on fsspec_, and inherits many useful behaviours from there, including integration with Dask, and the facility for key-value dict-like objects of the type used by zarr. @@ -19,12 +19,16 @@ Installation ------------ -The GCSFS library can be installed using ``conda`` or ``pip``: +The GCSFS library can be installed using ``conda``: .. code-block:: bash conda install -c conda-forge gcsfs - or + +or ``pip``: + +.. code-block:: bash + pip install gcsfs or by cloning the repository: @@ -50,7 +54,7 @@ ... print(f.read()) b'Hello, world' -(see also ``walk`` and ``glob``) +(see also :meth:`~gcsfs.core.GCSFileSystem.walk` and :meth:`~gcsfs.core.GCSFileSystem.glob`) Read with delimited blocks: @@ -128,7 +132,7 @@ storage_options={"token": "anon"}) This gives the chance to pass any credentials or other necessary -arguments needed to s3fs. +arguments needed to gcsfs. Async @@ -176,7 +180,7 @@ For further reference check `aiohttp proxy support`_. -.. _aiohttp proxy support: https://docs.aiohttp.org/en/stable/client_advanced.html?highlight=proxy#proxy-support +.. _aiohttp proxy support: https://docs.aiohttp.org/en/stable/client_advanced.html#proxy-support Contents diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/environment_gcsfs.yaml new/gcsfs-2023.3.0/environment_gcsfs.yaml --- old/gcsfs-2022.11.0/environment_gcsfs.yaml 1970-01-01 01:00:00.000000000 +0100 +++ new/gcsfs-2023.3.0/environment_gcsfs.yaml 2023-03-04 21:33:12.000000000 +0100 @@ -0,0 +1,21 @@ +name: gcsfs_test +channels: + - conda-forge +dependencies: + - aiohttp + - crcmod + - decorator + - fsspec + - fusepy<3 + - google-api-core + - google-api-python-client + - google-auth + - google-auth-oauthlib + - google-cloud-core + - libfuse<3 + - pytest + - pytest-timeout + - requests + - ujson + - pip: + - git+https://github.com/fsspec/filesystem_spec diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/_version.py new/gcsfs-2023.3.0/gcsfs/_version.py --- old/gcsfs-2022.11.0/gcsfs/_version.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/_version.py 2023-03-04 21:33:12.000000000 +0100 @@ -22,9 +22,9 @@ # setup.py/versioneer.py will grep for the variable names, so they must # each be defined on a line of their own. _version.py will just call # get_keywords(). - git_refnames = "2022.11.0" - git_full = "805d3fd359ba5189964f8804459653ce1eb4d38c" - git_date = "2022-11-09 21:57:38 -0500" + git_refnames = "2023.3.0" + git_full = "dda390af941b57b6911261e5c76d01cc3ddccb10" + git_date = "2023-03-04 15:33:12 -0500" keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords @@ -84,7 +84,7 @@ stderr=(subprocess.PIPE if hide_stderr else None), ) break - except EnvironmentError: + except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue @@ -94,7 +94,7 @@ return None, None else: if verbose: - print("unable to find command, tried %s" % (commands,)) + print(f"unable to find command, tried {commands}") return None, None stdout = p.communicate()[0].strip() if sys.version_info[0] >= 3: @@ -147,7 +147,7 @@ # _version.py. keywords = {} try: - f = open(versionfile_abs, "r") + f = open(versionfile_abs) for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) @@ -162,7 +162,7 @@ if mo: keywords["date"] = mo.group(1) f.close() - except EnvironmentError: + except OSError: pass return keywords @@ -186,11 +186,11 @@ if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -199,7 +199,7 @@ # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "". - tags = set([r for r in refs if re.search(r"\d", r)]) + tags = {r for r in refs if re.search(r"\d", r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -302,7 +302,7 @@ if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + pieces["error"] = "tag '{}' doesn't start with prefix '{}'".format( full_tag, tag_prefix, ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/checkers.py new/gcsfs-2023.3.0/gcsfs/checkers.py --- old/gcsfs-2022.11.0/gcsfs/checkers.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/checkers.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,9 +1,9 @@ -from base64 import b64encode import base64 -from typing import Optional +from base64 import b64encode from hashlib import md5 -from .retry import ChecksumError +from typing import Optional +from .retry import ChecksumError try: import crcmod diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/cli/gcsfuse.py new/gcsfs-2023.3.0/gcsfs/cli/gcsfuse.py --- old/gcsfs-2022.11.0/gcsfs/cli/gcsfuse.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/cli/gcsfuse.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,5 +1,6 @@ -import click import logging + +import click from fuse import FUSE from gcsfs.gcsfuse import GCSFS @@ -54,7 +55,7 @@ if verbose > 1: logging.basicConfig(level=logging.DEBUG, format=fmt) - print("Mounting bucket %s to directory %s" % (bucket, mount_point)) + print(f"Mounting bucket {bucket} to directory {mount_point}") print("foreground:", foreground, ", nothreads:", not threads) FUSE( GCSFS(bucket, token=token, project=project_id, nfiles=cache_files), diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/core.py new/gcsfs-2023.3.0/gcsfs/core.py --- old/gcsfs-2022.11.0/gcsfs/core.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/core.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- """ Google Cloud Storage pythonic interface """ import asyncio -import fsspec - import io import json import logging @@ -13,23 +10,27 @@ import re import warnings import weakref +from datetime import datetime +from urllib.parse import parse_qs +from urllib.parse import quote as quote_urllib +from urllib.parse import urlsplit -from fsspec.asyn import sync_wrapper, sync, AsyncFileSystem -from fsspec.utils import stringify_path, setup_logging +import fsspec +from fsspec.asyn import AsyncFileSystem, sync, sync_wrapper from fsspec.callbacks import NoOpCallback from fsspec.implementations.http import get_client -from .retry import retry_request, validate_response +from fsspec.utils import setup_logging, stringify_path + +from . import __version__ as version from .checkers import get_consistency_checker from .credentials import GoogleCredentials -from . import __version__ as version -from urllib.parse import quote as quote_urllib -from urllib.parse import parse_qs, urlsplit +from .retry import retry_request, validate_response logger = logging.getLogger("gcsfs") if "GCSFS_DEBUG" in os.environ: - setup_logging(logger=logger, level=os.environ["GCSFS_DEBUG"]) + setup_logging(logger=logger, level=os.getenv("GCSFS_DEBUG")) # client created 2018-01-16 @@ -48,7 +49,7 @@ "publicRead", "publicReadWrite", } -DEFAULT_PROJECT = os.environ.get("GCSFS_DEFAULT_PROJECT", "") +DEFAULT_PROJECT = os.getenv("GCSFS_DEFAULT_PROJECT", "") GCS_MIN_BLOCK_SIZE = 2**18 GCS_MAX_BLOCK_SIZE = 2**28 @@ -105,7 +106,7 @@ ------- valid http location """ - _emulator_location = os.environ.get("STORAGE_EMULATOR_HOST", None) + _emulator_location = os.getenv("STORAGE_EMULATOR_HOST", None) return ( _emulator_location if _emulator_location else "https://storage.googleapis.com"; ) @@ -148,7 +149,7 @@ metadata service, anonymous. - ``token='google_default'``, your default gcloud credentials will be used, which are typically established by doing ``gcloud login`` in a terminal. - - ``token=='cache'``, credentials from previously successful gcsfs + - ``token='cache'``, credentials from previously successful gcsfs authentication will be used (use this after "browser" auth succeeded) - ``token='anon'``, no authentication is performed, and you can only access data which is accessible to allUsers (in this case, the project and @@ -165,10 +166,10 @@ or a Credentials object. gcloud typically stores its tokens in locations such as ``~/.config/gcloud/application_default_credentials.json``, - `` ~/.config/gcloud/credentials``, or + ``~/.config/gcloud/credentials``, or ``~\AppData\Roaming\gcloud\credentials``, etc. - Specific methods, (eg. `ls`, `info`, ...) may return object details from GCS. + Specific methods, (eg. ``ls``, ``info``, ...) may return object details from GCS. These detailed listings include the [object resource](https://cloud.google.com/storage/docs/json_api/v1/objects#resource) @@ -198,8 +199,8 @@ created via other processes *will not* be visible to the GCSFileSystem until the cache refreshed. Calls to GCSFileSystem.open and calls to GCSFile are not effected by this cache. - In the default case the cache is never expired. This may be controlled via the `cache_timeout` - GCSFileSystem parameter or via explicit calls to `GCSFileSystem.invalidate_cache`. + In the default case the cache is never expired. This may be controlled via the ``cache_timeout`` + GCSFileSystem parameter or via explicit calls to ``GCSFileSystem.invalidate_cache``. Parameters ---------- @@ -224,11 +225,11 @@ secure_serialize: bool (deprecated) requester_pays : bool, or str default False Whether to use requester-pays requests. This will include your - project ID `project` in requests as the `userPorject`, and you'll be + project ID `project` in requests as the `userProject`, and you'll be billed for accessing data from requester-pays buckets. Optionally, pass a project-id here as a string to use that as the `userProject`. session_kwargs: dict - passed on to aiohttp.ClientSession; can contain, for example, + passed on to ``aiohttp.ClientSession``; can contain, for example, proxy settings. endpoint_url: str If given, use this URL (format protocol://host:port , *without* any @@ -303,12 +304,6 @@ self.credentials = GoogleCredentials(project, access, token) - if not self.asynchronous: - self._session = sync( - self.loop, get_client, timeout=self.timeout, **self.session_kwargs - ) - weakref.finalize(self, self.close_session, self.loop, self._session) - @property def _location(self): return self._endpoint or _location() @@ -335,6 +330,7 @@ async def _set_session(self): if self._session is None: self._session = await get_client(**self.session_kwargs) + weakref.finalize(self, self.close_session, self.loop, self._session) return self._session @property @@ -741,6 +737,18 @@ rmdir = sync_wrapper(_rmdir) + def modified(self, path): + return self._parse_timestamp(self.info(path)["updated"]) + + def created(self, path): + return self._parse_timestamp(self.info(path)["timeCreated"]) + + def _parse_timestamp(self, timestamp): + assert timestamp.endswith("Z") + timestamp = timestamp[:-1] + timestamp = timestamp + "0" * (6 - len(timestamp.rsplit(".", 1)[1])) + return datetime.fromisoformat(timestamp + "+00:00") + async def _info(self, path, generation=None, **kwargs): """File information about this path.""" path = self._strip_protocol(path) @@ -815,14 +823,21 @@ prefix = path[:ind].split("/")[-1] return await super()._glob(path, prefix=prefix, **kwargs) - async def _ls(self, path, detail=False, prefix="", **kwargs): + async def _ls(self, path, detail=False, prefix="", versions=False, **kwargs): """List objects under the given '/{bucket}/{prefix} path.""" path = self._strip_protocol(path).rstrip("/") if path in ["/", ""]: out = await self._list_buckets() else: - out = await self._list_objects(path, prefix=prefix) + out = [] + for entry in await self._list_objects( + path, prefix=prefix, versions=versions + ): + if versions and "generation" in entry: + entry = entry.copy() + entry["name"] = f"{entry['name']}#{entry['generation']}" + out.append(entry) if detail: return out @@ -838,7 +853,7 @@ self._location, bucket, object, - "&generation={}".format(generation) if generation else "", + f"&generation={generation}" if generation else "", ) async def _cat_file(self, path, start=None, end=None, **kwargs): @@ -872,7 +887,7 @@ fake-gcs-server:latest does not seem to support this. Parameters - --------- + ---------- content_type: str If not None, set the content-type to this value content_encoding: str @@ -886,6 +901,7 @@ - content_encoding - content_language - custom_time + More info: https://cloud.google.com/storage/docs/metadata#mutable kw_args: key-value pairs like field="value" or field=None @@ -1164,7 +1180,7 @@ async def _isdir(self, path): try: return (await self._info(path))["type"] == "directory" - except IOError: + except OSError: return False async def _find( @@ -1204,7 +1220,9 @@ "size": 0, } - cache_entries.setdefault(parent, []).append(previous) + listing = cache_entries.setdefault(parent, []) + if previous not in listing: + listing.append(previous) previous = dirs[parent] parent = self._parent(parent) @@ -1229,7 +1247,7 @@ self, rpath, lpath, *args, headers=None, callback=None, **kwargs ): consistency = kwargs.pop("consistency", self.consistency) - + await self._set_session() async with self.session.get( url=rpath, params=self._get_params(kwargs), @@ -1370,13 +1388,15 @@ """ from google.cloud import storage - bucket, key = self.split_path(path) + bucket, key, generation = self.split_path(path) client = storage.Client( credentials=self.credentials.credentials, project=self.project ) bucket = client.bucket(bucket) blob = bucket.blob(key) - return blob.generate_signed_url(expiration=expiration, **kwargs) + return blob.generate_signed_url( + expiration=expiration, generation=generation, **kwargs + ) GoogleCredentials.load_tokens() @@ -1662,7 +1682,7 @@ range = "bytes %i-%i/%i" % (offset, offset + l - 1, size) head["Content-Range"] = range head.update({"Content-Type": content_type, "Content-Length": str(l)}) - headers, txt = await fs._call("POST", location, headers=head, data=data) + headers, txt = await fs._call("POST", location, headers=head, data=io.BytesIO(data)) if "Range" in headers: end = int(headers["Range"].split("-")[1]) shortfall = (offset + l - 1) - end @@ -1719,11 +1739,7 @@ template = ( "--==0==" "\nContent-Type: application/json; charset=UTF-8" - "\n\n" - + metadata - + "\n--==0==" - + "\nContent-Type: {0}".format(content_type) - + "\n\n" + "\n\n" + metadata + "\n--==0==" + f"\nContent-Type: {content_type}" + "\n\n" ) data = template.encode() + datain + b"\n--==0==--" @@ -1732,7 +1748,7 @@ path, uploadType="multipart", headers={"Content-Type": 'multipart/related; boundary="==0=="'}, - data=data, + data=io.BytesIO(data), json_out=True, ) checker.update(datain) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/credentials.py new/gcsfs-2023.3.0/gcsfs/credentials.py --- old/gcsfs-2022.11.0/gcsfs/credentials.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/credentials.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,21 +1,20 @@ +import json +import logging +import os +import pickle import textwrap +import threading +import warnings import google.auth as gauth import google.auth.compute_engine import google.auth.credentials import google.auth.exceptions +import requests +from google.auth.transport.requests import Request +from google.oauth2 import service_account from google.oauth2.credentials import Credentials from google_auth_oauthlib.flow import InstalledAppFlow -from google.oauth2 import service_account -from google.auth.transport.requests import Request -import json -import requests -import os -import pickle -import requests -import threading -import warnings -import logging logger = logging.getLogger("gcsfs.credentials") @@ -154,7 +153,8 @@ # TODO: catch specific exceptions # some other kind of token file # will raise exception if is not json - token = json.load(open(token)) + with open(token) as data: + token = json.load(data) if isinstance(token, dict): credentials = self._dict_to_credentials(token) elif isinstance(token, google.auth.credentials.Credentials): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/retry.py new/gcsfs-2023.3.0/gcsfs/retry.py --- old/gcsfs-2022.11.0/gcsfs/retry.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/retry.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,13 +1,12 @@ import asyncio -from decorator import decorator import json import logging import random - -import requests.exceptions -import google.auth.exceptions import aiohttp.client_exceptions +import google.auth.exceptions +import requests.exceptions +from decorator import decorator logger = logging.getLogger("gcsfs") @@ -28,7 +27,7 @@ self.message = "" self.code = None # Call the base class constructor with the parameters it needs - super(HttpError, self).__init__(self.message) + super().__init__(self.message) class ChecksumError(Exception): @@ -93,11 +92,11 @@ msg = content if status == 403: - raise IOError("Forbidden: %s\n%s" % (path, msg)) + raise OSError(f"Forbidden: {path}\n{msg}") elif status == 502: raise requests.exceptions.ProxyError() elif "invalid" in str(msg): - raise ValueError("Bad Request: %s\n%s" % (path, msg)) + raise ValueError(f"Bad Request: {path}\n{msg}") elif error: raise HttpError(error) elif status: @@ -141,12 +140,10 @@ logger.debug("Request returned 404, no retries.") raise e if retry == retries - 1: - logger.exception( - "%s out of retries on exception: %s" % (func.__name__, e) - ) + logger.exception(f"{func.__name__} out of retries on exception: {e}") raise e if is_retriable(e): - logger.debug("%s retrying after exception: %s" % (func.__name__, e)) + logger.debug(f"{func.__name__} retrying after exception: {e}") continue - logger.exception("%s non-retriable exception: %s" % (func.__name__, e)) + logger.exception(f"{func.__name__} non-retriable exception: {e}") raise e diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/tests/conftest.py new/gcsfs-2023.3.0/gcsfs/tests/conftest.py --- old/gcsfs-2022.11.0/gcsfs/tests/conftest.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/tests/conftest.py 2023-03-04 21:33:12.000000000 +0100 @@ -58,7 +58,7 @@ def docker_gcs(): if "STORAGE_EMULATOR_HOST" in os.environ: # assume using real API or otherwise have a server already set up - yield os.environ["STORAGE_EMULATOR_HOST"] + yield os.getenv("STORAGE_EMULATOR_HOST") return container = "gcsfs_test" cmd = ( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/tests/settings.py new/gcsfs-2023.3.0/gcsfs/tests/settings.py --- old/gcsfs-2022.11.0/gcsfs/tests/settings.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/tests/settings.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,5 +1,5 @@ import os -TEST_BUCKET = os.environ.get("GCSFS_TEST_BUCKET", "gcsfs_test") -TEST_PROJECT = os.environ.get("GCSFS_TEST_PROJECT", "project") +TEST_BUCKET = os.getenv("GCSFS_TEST_BUCKET", "gcsfs_test") +TEST_PROJECT = os.getenv("GCSFS_TEST_PROJECT", "project") TEST_REQUESTER_PAYS_BUCKET = "gcsfs_test_req_pay" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/tests/test_checkers.py new/gcsfs-2023.3.0/gcsfs/tests/test_checkers.py --- old/gcsfs-2022.11.0/gcsfs/tests/test_checkers.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/tests/test_checkers.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,10 +1,11 @@ -from gcsfs.retry import ChecksumError -from gcsfs.checkers import Crc32cChecker, MD5Checker, SizeChecker, crcmod -from hashlib import md5 import base64 +from hashlib import md5 import pytest +from gcsfs.checkers import Crc32cChecker, MD5Checker, SizeChecker, crcmod +from gcsfs.retry import ChecksumError + def google_response_from_data(expected_data: bytes, actual_data=None): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/tests/test_core.py new/gcsfs-2023.3.0/gcsfs/tests/test_core.py --- old/gcsfs-2022.11.0/gcsfs/tests/test_core.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/tests/test_core.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,32 +1,29 @@ -# -*- coding: utf-8 -*- - +import datetime import io +import os from builtins import FileNotFoundError from itertools import chain from unittest import mock -from urllib.parse import urlparse, parse_qs, unquote +from urllib.parse import parse_qs, unquote, urlparse from uuid import uuid4 +import fsspec.core import pytest import requests - -from fsspec.utils import seek_delimiter from fsspec.asyn import sync +from fsspec.utils import seek_delimiter -from gcsfs.tests.settings import TEST_BUCKET, TEST_PROJECT, TEST_REQUESTER_PAYS_BUCKET -from gcsfs.tests.conftest import ( - a, - allfiles, - b, - csv_files, - files, - text_files, -) -from gcsfs.tests.utils import tempdir, tmpfile -from gcsfs.core import GCSFileSystem, quote -from gcsfs.credentials import GoogleCredentials import gcsfs.checkers +import gcsfs.tests.settings from gcsfs import __version__ as version +from gcsfs.core import GCSFileSystem, quote +from gcsfs.credentials import GoogleCredentials +from gcsfs.tests.conftest import a, allfiles, b, csv_files, files, text_files +from gcsfs.tests.utils import tempdir, tmpfile + +TEST_BUCKET = gcsfs.tests.settings.TEST_BUCKET +TEST_PROJECT = gcsfs.tests.settings.TEST_PROJECT +TEST_REQUESTER_PAYS_BUCKET = gcsfs.tests.settings.TEST_REQUESTER_PAYS_BUCKET def test_simple(gcs): @@ -128,6 +125,10 @@ gcs.touch(a) assert gcs.info(a) == gcs.ls(a, detail=True)[0] + today = datetime.date.today().isoformat() + assert gcs.created(a).isoformat().startswith(today) + assert gcs.modified(a).isoformat().startswith(today) + def test_ls2(gcs): assert TEST_BUCKET + "/" in gcs.ls("") @@ -167,10 +168,10 @@ gcs.touch(b) L = gcs.ls(TEST_BUCKET + "/tmp/test", False) - assert set(L) == set([a, b]) + assert set(L) == {a, b} L_d = gcs.ls(TEST_BUCKET + "/tmp/test", True) - assert set(d["name"] for d in L_d) == set([a, b]) + assert {d["name"] for d in L_d} == {a, b} def test_rm(gcs): @@ -1077,7 +1078,7 @@ fn2 = unquote(fn) gcs.touch(fn2) assert gcs.cat(fn2) != data - assert set(gcs.ls(parent)) == set([fn, fn2]) + assert set(gcs.ls(parent)) == {fn, fn2} @pytest.mark.parametrize( @@ -1193,6 +1194,24 @@ assert gcs_versioned.cat(b) == b"v1" +def test_ls_versioned(gcs_versioned): + import posixpath + + with gcs_versioned.open(a, "wb") as wo: + wo.write(b"v1") + v1 = gcs_versioned.info(a)["generation"] + with gcs_versioned.open(a, "wb") as wo: + wo.write(b"v2") + v2 = gcs_versioned.info(a)["generation"] + dpath = posixpath.dirname(a) + versions = {f"{a}#{v1}", f"{a}#{v2}"} + assert versions == set(gcs_versioned.ls(dpath, versions=True)) + assert versions == { + entry["name"] for entry in gcs_versioned.ls(dpath, detail=True, versions=True) + } + assert gcs_versioned.ls(TEST_BUCKET, versions=True) == ["gcsfs_test/tmp"] + + def test_find_versioned(gcs_versioned): with gcs_versioned.open(a, "wb") as wo: wo.write(b"v1") @@ -1200,7 +1219,148 @@ with gcs_versioned.open(a, "wb") as wo: wo.write(b"v2") v2 = gcs_versioned.info(a)["generation"] - assert {f"{a}#{v1}", f"{a}#{v2}"} == set(gcs_versioned.find(a, versions=True)) - assert {f"{a}#{v1}", f"{a}#{v2}"} == set( - gcs_versioned.find(a, detail=True, versions=True) - ) + versions = {f"{a}#{v1}", f"{a}#{v2}"} + assert versions == set(gcs_versioned.find(a, versions=True)) + assert versions == set(gcs_versioned.find(a, detail=True, versions=True)) + + +def test_cp_directory_recursive(gcs): + src = TEST_BUCKET + "/src" + src_file = src + "/file" + gcs.mkdir(src) + gcs.touch(src_file) + + target = TEST_BUCKET + "/target" + + # cp without slash + assert not gcs.exists(target) + for loop in range(2): + gcs.cp(src, target, recursive=True) + assert gcs.isdir(target) + + if loop == 0: + correct = [target + "/file"] + assert gcs.find(target) == correct + else: + correct = [target + "/file", target + "/src/file"] + assert sorted(gcs.find(target)) == correct + + gcs.rm(target, recursive=True) + + # cp with slash + assert not gcs.exists(target) + for loop in range(2): + gcs.cp(src + "/", target, recursive=True) + assert gcs.isdir(target) + correct = [target + "/file"] + assert gcs.find(target) == correct + + +def test_get_directory_recursive(gcs): + src = TEST_BUCKET + "/src" + src_file = src + "/file" + gcs.mkdir(src) + gcs.touch(src_file) + + with tempdir() as tmpdir: + target = os.path.join(tmpdir, "target") + target_fs = fsspec.filesystem("file") + + # get without slash + assert not target_fs.exists(target) + for loop in range(2): + gcs.get(src, target, recursive=True) + assert target_fs.isdir(target) + + if loop == 0: + assert target_fs.find(target) == [os.path.join(target, "file")] + else: + assert sorted(target_fs.find(target)) == [ + os.path.join(target, "file"), + os.path.join(target, "src", "file"), + ] + + target_fs.rm(target, recursive=True) + + # get with slash + assert not target_fs.exists(target) + for loop in range(2): + gcs.get(src + "/", target, recursive=True) + assert target_fs.isdir(target) + assert target_fs.find(target) == [os.path.join(target, "file")] + + +def test_put_directory_recursive(gcs): + with tempdir() as tmpdir: + src = os.path.join(tmpdir, "src") + src_file = os.path.join(src, "file") + + source_fs = fsspec.filesystem("file") + source_fs.mkdir(src) + source_fs.touch(src_file) + + target = TEST_BUCKET + "/target" + + # put without slash + assert not gcs.exists(target) + for loop in range(2): + gcs.put(src, target, recursive=True) + assert gcs.isdir(target) + + if loop == 0: + assert gcs.find(target) == [target + "/file"] + else: + assert sorted(gcs.find(target)) == [ + target + "/file", + target + "/src/file", + ] + + gcs.rm(target, recursive=True) + + # put with slash + assert not gcs.exists(target) + for loop in range(2): + gcs.put(src + "/", target, recursive=True) + assert gcs.isdir(target) + assert gcs.find(target) == [target + "/file"] + + +def test_cp_two_files(gcs): + src = TEST_BUCKET + "/src" + file0 = src + "/file0" + file1 = src + "/file1" + gcs.mkdir(src) + gcs.touch(file0) + gcs.touch(file1) + + target = TEST_BUCKET + "/target" + assert not gcs.exists(target) + + gcs.cp([file0, file1], target) + + assert gcs.isdir(target) + assert sorted(gcs.find(target)) == [ + target + "/file0", + target + "/file1", + ] + + +def test_multiglob(gcs): + # #530 + root = TEST_BUCKET + + ggparent = root + "/t1" + gparent = ggparent + "/t2" + parent = gparent + "/t3" + leaf1 = parent + "/foo.txt" + leaf2 = parent + "/bar.txt" + leaf3 = parent + "/baz.txt" + + gcs.touch(leaf1) + gcs.touch(leaf2) + gcs.touch(leaf3) + gcs.invalidate_cache() + + assert gcs.ls(gparent, detail=False) == [f"{root}/t1/t2/t3"] + gcs.glob(ggparent + "/") + assert gcs.ls(gparent, detail=False) == [f"{root}/t1/t2/t3"] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/tests/test_fuse.py new/gcsfs-2023.3.0/gcsfs/tests/test_fuse.py --- old/gcsfs-2022.11.0/gcsfs/tests/test_fuse.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/tests/test_fuse.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,39 +1,63 @@ +import logging import os +import sys +import tempfile +import threading +import time +from functools import partial import pytest -import tempfile - -fuse = pytest.importorskip("fuse") -from fsspec.fuse import run from gcsfs.tests.settings import TEST_BUCKET -import threading -import time -def test_fuse(gcs): [email protected](180) [email protected] +def fsspec_fuse_run(): + """Fixture catches other errors on fuse import.""" + try: + _fuse = pytest.importorskip("fuse") # noqa + + from fsspec.fuse import run as _fsspec_fuse_run + + return _fsspec_fuse_run + except Exception as error: + logging.debug("Error importing fuse: %s", error) + pytest.skip("Error importing fuse.") + + [email protected](sys.version_info < (3, 9), reason="Test fuse causes hang.") [email protected](reason="Failing test not previously tested.") [email protected](180) +def test_fuse(gcs, fsspec_fuse_run): mountpath = tempfile.mkdtemp() - th = threading.Thread(target=lambda: run(gcs, TEST_BUCKET + "/", mountpath)) + _run = partial(fsspec_fuse_run, gcs, TEST_BUCKET + "/", mountpath) + th = threading.Thread(target=_run) th.daemon = True th.start() time.sleep(5) timeout = 20 - while True: + n = 40 + for i in range(n): + logging.debug(f"Attempt # {i+1}/{n} to create lock file.") try: open(os.path.join(mountpath, "lock"), "w").close() os.remove(os.path.join(mountpath, "lock")) break - except: # noqa: E722 + except Exception as error: # noqa: E722 + logging.debug("Error: %s", error) time.sleep(0.5) timeout -= 0.5 assert timeout > 0 + else: + raise AssertionError(f"Attempted lock file failed after {n} attempts.") with open(os.path.join(mountpath, "hello"), "w") as f: # NB this is in TEXT mode f.write("hello") files = os.listdir(mountpath) assert "hello" in files - with open(os.path.join(mountpath, "hello"), "r") as f: + with open(os.path.join(mountpath, "hello")) as f: # NB this is in TEXT mode assert f.read() == "hello" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/tests/test_manyopens.py new/gcsfs-2023.3.0/gcsfs/tests/test_manyopens.py --- old/gcsfs-2022.11.0/gcsfs/tests/test_manyopens.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/tests/test_manyopens.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test helper to open the same file many times. @@ -9,8 +8,9 @@ Ideally you should see nothing, just the attempt count go up until we're done. """ -from __future__ import print_function + import sys + import gcsfs diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/tests/test_mapping.py new/gcsfs-2023.3.0/gcsfs/tests/test_mapping.py --- old/gcsfs-2022.11.0/gcsfs/tests/test_mapping.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/tests/test_mapping.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,7 +1,8 @@ import pytest + from gcsfs.tests.settings import TEST_BUCKET -root = TEST_BUCKET + "/mapping" +MAPPING_ROOT = TEST_BUCKET + "/mapping" def test_api(): @@ -12,7 +13,7 @@ def test_map_simple(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) assert not d assert list(d) == list(d.keys()) == [] @@ -21,12 +22,12 @@ def test_map_default_gcsfilesystem(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) assert d.fs is gcs def test_map_errors(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) with pytest.raises(KeyError): d["nonexistent"] try: @@ -36,7 +37,7 @@ def test_map_with_data(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d["x"] = b"123" assert list(d) == list(d.keys()) == ["x"] assert list(d.values()) == [b"123"] @@ -44,7 +45,7 @@ assert d["x"] == b"123" assert bool(d) - assert gcs.find(root) == [TEST_BUCKET + "/mapping/x"] + assert gcs.find(MAPPING_ROOT) == [TEST_BUCKET + "/mapping/x"] d["x"] = b"000" assert d["x"] == b"000" @@ -57,7 +58,7 @@ def test_map_complex_keys(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d[1] = b"hello" assert d[1] == b"hello" del d[1] @@ -73,7 +74,7 @@ def test_map_clear_empty(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d.clear() assert list(d) == [] d[1] = b"1" @@ -84,7 +85,7 @@ def test_map_pickle(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d["x"] = b"1" assert d["x"] == b"1" @@ -98,14 +99,14 @@ def test_map_array(gcs): from array import array - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d["x"] = array("B", [65] * 1000) assert d["x"] == b"A" * 1000 def test_map_bytearray(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d["x"] = bytearray(b"123") assert d["x"] == b"123" @@ -134,7 +135,7 @@ def test_map_pickle(gcs): import pickle - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d["x"] = b"1234567890" b = pickle.dumps(d) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/tests/test_retry.py new/gcsfs-2023.3.0/gcsfs/tests/test_retry.py --- old/gcsfs-2022.11.0/gcsfs/tests/test_retry.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/tests/test_retry.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,10 +1,11 @@ import os + +import pytest import requests from requests.exceptions import ProxyError -import pytest -from gcsfs.tests.settings import TEST_BUCKET from gcsfs.retry import HttpError, is_retriable, validate_response +from gcsfs.tests.settings import TEST_BUCKET from gcsfs.tests.utils import tmpfile diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/gcsfs/tests/utils.py new/gcsfs-2023.3.0/gcsfs/tests/utils.py --- old/gcsfs-2022.11.0/gcsfs/tests/utils.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/gcsfs/tests/utils.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,7 +1,7 @@ -from contextlib import contextmanager import os import shutil import tempfile +from contextlib import contextmanager @contextmanager diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/requirements.txt new/gcsfs-2023.3.0/requirements.txt --- old/gcsfs-2022.11.0/requirements.txt 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/requirements.txt 2023-03-04 21:33:12.000000000 +0100 @@ -1,7 +1,7 @@ +aiohttp!=4.0.0a0, !=4.0.0a1 +decorator>4.1.2 +fsspec==2023.3.0 google-auth>=1.2 google-auth-oauthlib google-cloud-storage requests -decorator>4.1.2 -fsspec==2022.11.0 -aiohttp!=4.0.0a0, !=4.0.0a1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/setup.cfg new/gcsfs-2023.3.0/setup.cfg --- old/gcsfs-2022.11.0/setup.cfg 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/setup.cfg 2023-03-04 21:33:12.000000000 +0100 @@ -11,18 +11,30 @@ [flake8] exclude = versioneer.py,docs/source/conf.py ignore = - E20, # Extra space in brackets - E231,E241, # Multiple spaces around "," - E26, # Comments - E4, # Import formatting - E721, # Comparing types instead of isinstance - E731, # Assigning lambda expression - E741, # Ambiguous variable names - W503, # line break before binary operator - W504, # line break after binary operator - F811, # redefinition of unused 'loop' from line 10 + # Extra space in brackets + E20, + # Multiple spaces around "," + E231,E241, + # Comments + E26, + # Import formatting + E4, + # Comparing types instead of isinstance + E721, + # Assigning lambda expression + E731, + # Ambiguous variable names + E741, + # line break before binary operator + W503, + # line break after binary operator + W504, + # redefinition of unused 'loop' from line 10 + F811, max-line-length = 120 [tool:pytest] addopts = - --color=yes + --color=yes --timeout=600 +log_cli = false +log_cli_level = DEBUG diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.11.0/setup.py new/gcsfs-2023.3.0/setup.py --- old/gcsfs-2022.11.0/setup.py 2022-11-10 03:57:38.000000000 +0100 +++ new/gcsfs-2023.3.0/setup.py 2023-03-04 21:33:12.000000000 +0100 @@ -1,9 +1,10 @@ #!/usr/bin/env python import os + from setuptools import setup -import versioneer +import versioneer setup( name="gcsfs", @@ -19,9 +20,10 @@ "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], keywords=["google-cloud-storage", "gcloud", "file-system"], packages=["gcsfs", "gcsfs.cli"], @@ -30,6 +32,6 @@ open("README.rst").read() if os.path.exists("README.rst") else "" ), extras_require={"gcsfuse": ["fusepy"], "crc": ["crcmod"]}, - python_requires=">=3.7", + python_requires=">=3.8", zip_safe=False, )
