Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-gcsfs for openSUSE:Factory checked in at 2022-07-04 11:32:45 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-gcsfs (Old) and /work/SRC/openSUSE:Factory/.python-gcsfs.new.1548 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-gcsfs" Mon Jul 4 11:32:45 2022 rev:11 rq:986479 version:2022.5.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-gcsfs/python-gcsfs.changes 2022-04-28 23:07:49.636678292 +0200 +++ /work/SRC/openSUSE:Factory/.python-gcsfs.new.1548/python-gcsfs.changes 2022-07-04 11:32:50.764020334 +0200 @@ -1,0 +2,12 @@ +Sat Jul 2 18:49:11 UTC 2022 - Ben Greiner <[email protected]> + +- Enable the test suite with fake-gcs-server running + +------------------------------------------------------------------- +Wed Jun 29 10:18:06 UTC 2022 - Ben Greiner <[email protected]> + +- Update to 2022.5.0 + * invalidate listings cache for simple put/pipe (#474) + * conform _mkdir and _cat_file to upstream (#471) + +------------------------------------------------------------------- Old: ---- gcsfs-2022.3.0-gh.tar.gz New: ---- gcsfs-2022.5.0-gh.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-gcsfs.spec ++++++ --- /var/tmp/diff_new_pack.D5WsI7/_old 2022-07-04 11:32:51.688021822 +0200 +++ /var/tmp/diff_new_pack.D5WsI7/_new 2022-07-04 11:32:51.696021835 +0200 @@ -17,11 +17,9 @@ %{?!python_module:%define python_module() python3-%{**}} -# the test suite moved to a docker simulator which we cannot run inside an obs environment -%bcond_with fulltest %define skip_python2 1 Name: python-gcsfs -Version: 2022.3.0 +Version: 2022.5.0 Release: 0 Summary: Filesystem interface over GCS License: BSD-3-Clause @@ -44,9 +42,7 @@ Suggests: python-crcmod BuildArch: noarch # SECTION test requirements -# always import in order to detect dependency breakages at build time BuildRequires: %{python_module aiohttp} -BuildRequires: %{python_module black} BuildRequires: %{python_module click} BuildRequires: %{python_module decorator > 4.1.2} BuildRequires: %{python_module fsspec == %{version}} @@ -56,8 +52,10 @@ BuildRequires: %{python_module google-auth >= 1.2} BuildRequires: %{python_module google-auth-oauthlib} BuildRequires: %{python_module google-cloud-storage} +BuildRequires: %{python_module pytest-timeout} BuildRequires: %{python_module pytest} BuildRequires: %{python_module requests} +BuildRequires: fake-gcs-server # /SECTION %python_subpackages @@ -87,33 +85,21 @@ %python_expand %fdupes %{buildroot}%{$python_sitelib} %check -## # Tests test_map_simple, test_map_with_data and test_map_clear_empty require a network connection -## %%pytest -k "not network" gcsfs/tests -## # export GCSFS_RECORD_MODE=none -## # export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json -## # %%%%pytest -rfEs -## -## - name: Install dependencies -## shell: bash -l {0} -## run: | -## pip install git+https://github.com/fsspec/filesystem_spec --no-deps -## conda list -## conda --version -## - name: Install -## shell: bash -l {0} -## run: pip install .[crc] -## -## - name: Run Tests -## shell: bash -l {0} -## run: | -## export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json -## py.test -vv gcsfs +# https://github.com/fsspec/gcsfs/blob/main/docs/source/developer.rst +fake-gcs-server \ + -backend memory \ + -scheme http \ + -public-host http://localhost:4443 \ + -external-url http://localhost:4443 \ + >$PWD/fake-gcs-server.stdout 2>$PWD/fake-gcs-server.stderr & +trap "jobs; kill %1 || true" EXIT +export STORAGE_EMULATOR_HOST=http://localhost:4443 export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json -%if %{with fulltest} -%pytest -rfEs -%else -%pytest -rfEs -k test_checkers -%endif +# Don't test fuse, it hangs on mounting inside obs +donttest="test_fuse" +# finds an existing path on the non-first multiflavor test runs" +donttest+=" or test_mkdir_with_path" +%pytest -rfEs -k "not ($donttest)" %files %{python_files} %doc README.rst ++++++ gcsfs-2022.3.0-gh.tar.gz -> gcsfs-2022.5.0-gh.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.3.0/docs/source/changelog.rst new/gcsfs-2022.5.0/docs/source/changelog.rst --- old/gcsfs-2022.3.0/docs/source/changelog.rst 2022-04-04 16:36:39.000000000 +0200 +++ new/gcsfs-2022.5.0/docs/source/changelog.rst 2022-05-20 16:37:06.000000000 +0200 @@ -1,6 +1,12 @@ Changelog ========= +2022.5.0 +-------- + +* invalidate listings cache for simple put/pipe (#474) +* conform _mkdir and _cat_file to upstream (#471) + 2022.3.0 -------- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.3.0/gcsfs/_version.py new/gcsfs-2022.5.0/gcsfs/_version.py --- old/gcsfs-2022.3.0/gcsfs/_version.py 2022-04-04 16:36:39.000000000 +0200 +++ new/gcsfs-2022.5.0/gcsfs/_version.py 2022-05-20 16:37:06.000000000 +0200 @@ -22,9 +22,9 @@ # setup.py/versioneer.py will grep for the variable names, so they must # each be defined on a line of their own. _version.py will just call # get_keywords(). - git_refnames = "2022.3.0" - git_full = "e4c16db7f59e0f243965f015e95996f9e7fe9665" - git_date = "2022-04-04 10:36:39 -0400" + git_refnames = "2022.5.0" + git_full = "58323d27ef7bdae8d81201b86d8a196d0a9632c8" + git_date = "2022-05-20 10:37:06 -0400" keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.3.0/gcsfs/core.py new/gcsfs-2022.5.0/gcsfs/core.py --- old/gcsfs-2022.3.0/gcsfs/core.py 2022-04-04 16:36:39.000000000 +0200 +++ new/gcsfs-2022.5.0/gcsfs/core.py 2022-05-20 16:37:06.000000000 +0200 @@ -605,17 +605,23 @@ async def _mkdir( self, - bucket, + path, acl="projectPrivate", default_acl="bucketOwnerFullControl", location=None, + create_parents=True, + **kwargs, ): """ New bucket + If path is more than just a bucket, will create bucket if create_parents=True; + otherwise is a noop. If create_parents is False and bucket does not exist, + will produce FileNotFFoundError. + Parameters ---------- - bucket: str + path: str bucket name. If contains '/' (i.e., looks like subdir), will have no effect because GCS doesn't have real directories. acl: string, one of bACLs @@ -627,11 +633,18 @@ If not provided, defaults to `self.default_location`. You can find a list of all available locations here: https://cloud.google.com/storage/docs/locations#available-locations + create_parents: bool + If True, creates the bucket in question, if it doesn't already exist """ + bucket, object = self.split_path(path) if bucket in ["", "/"]: raise ValueError("Cannot create root bucket") - if "/" in bucket: + if "/" in path and create_parents and await self._exists(bucket): + # nothing to do return + if "/" in path and not create_parents and not await self._exists(bucket): + raise FileNotFoundError(bucket) + json_data = {"name": bucket} location = location or self.default_location if location: @@ -758,7 +771,7 @@ object = quote_plus(object) return u.format(self._location, bucket, object) - async def _cat_file(self, path, start=None, end=None): + async def _cat_file(self, path, start=None, end=None, **kwargs): """Simple one-shot get of file data""" u2 = self.url(path) if start or end: @@ -987,7 +1000,7 @@ size = len(data) out = None if size < 5 * 2**20: - return await simple_upload( + location = await simple_upload( self, bucket, key, data, metadata, consistency, content_type ) else: @@ -998,10 +1011,12 @@ self, location, bit, offset, size, content_type ) - checker = get_consistency_checker(consistency) - checker.update(data) - checker.validate_json_response(out) + checker = get_consistency_checker(consistency) + checker.update(data) + checker.validate_json_response(out) + self.invalidate_cache(self._parent(path)) + return location async def _put_file( self, @@ -1037,7 +1052,7 @@ content_type=content_type, ) callback.absolute_update(size) - return None + else: location = await initiate_upload( self, bucket, key, content_type, metadata @@ -1054,7 +1069,8 @@ callback.absolute_update(offset) checker.update(bit) - checker.validate_json_response(out) + checker.validate_json_response(out) + self.invalidate_cache(self._parent(rpath)) async def _isdir(self, path): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.3.0/gcsfs/tests/test_core.py new/gcsfs-2022.5.0/gcsfs/tests/test_core.py --- old/gcsfs-2022.3.0/gcsfs/tests/test_core.py 2022-04-04 16:36:39.000000000 +0200 +++ new/gcsfs-2022.5.0/gcsfs/tests/test_core.py 2022-05-20 16:37:06.000000000 +0200 @@ -917,6 +917,38 @@ gcs.info(f"gs://{TEST_BUCKET}/b") +def test_pipe_small_cache_validity(gcs): + folder = f"{TEST_BUCKET}/{str(uuid4())}" + + gcs.pipe(f"gs://{folder}/a/file.txt", b"") + + assert gcs.ls(f"gs://{folder}") == [f"{folder}/a"] + + gcs.pipe(f"gs://{folder}/b/file.txt", b"") + + ls_res = gcs.ls(f"gs://{folder}") + assert len(ls_res) == 2 + assert f"{folder}/b" in ls_res + + +def test_put_small_cache_validity(gcs): + folder = f"{TEST_BUCKET}/{str(uuid4())}" + + gcs.pipe(f"gs://{folder}/a/file.txt", b"") + + assert gcs.ls(f"gs://{folder}") == [f"{folder}/a"] + + with tmpfile() as fn: + with open(fn, "w") as f: + f.write("") + + gcs.put(fn, f"gs://{folder}/b/file.txt", b"") + + ls_res = gcs.ls(f"gs://{folder}") + assert len(ls_res) == 2 + assert f"{folder}/b" in ls_res + + def test_pseudo_dir_find(gcs): gcs.rm(f"{TEST_BUCKET}/*", recursive=True) gcs.touch(f"{TEST_BUCKET}/a/b/file") @@ -1035,3 +1067,15 @@ out3 = gcs.info(f"{TEST_BUCKET}/placeholder/") assert out2 == out3 assert out2["type"] == "directory" + + +def test_mkdir_with_path(gcs): + with pytest.raises(FileNotFoundError): + gcs.mkdir("new/path", create_parents=False) + assert not gcs.exists("new") + gcs.mkdir("new/path", create_parents=True) + assert gcs.exists("new") + + # these lines do nothing, but should not fail + gcs.mkdir("new/path", create_parents=False) + gcs.mkdir("new/path", create_parents=True) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/gcsfs-2022.3.0/requirements.txt new/gcsfs-2022.5.0/requirements.txt --- old/gcsfs-2022.3.0/requirements.txt 2022-04-04 16:36:39.000000000 +0200 +++ new/gcsfs-2022.5.0/requirements.txt 2022-05-20 16:37:06.000000000 +0200 @@ -3,5 +3,5 @@ google-cloud-storage requests decorator>4.1.2 -fsspec==2022.3.0 +fsspec==2022.5.0 aiohttp<4
