Nilesh Patra pushed to branch upstream at Debian Med / python-xopen
Commits: 01d726c2 by Nilesh Patra at 2021-10-19T15:45:11+05:30 New upstream version 1.2.1 - - - - - 5 changed files: - PKG-INFO - src/xopen.egg-info/PKG-INFO - src/xopen/__init__.py - src/xopen/_version.py - tests/test_xopen.py Changes: ===================================== PKG-INFO ===================================== @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: xopen -Version: 1.2.0 +Version: 1.2.1 Summary: Open compressed files transparently Home-page: https://github.com/pycompression/xopen/ Author: Marcel Martin et al. ===================================== src/xopen.egg-info/PKG-INFO ===================================== @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: xopen -Version: 1.2.0 +Version: 1.2.1 Summary: Open compressed files transparently Home-page: https://github.com/pycompression/xopen/ Author: Marcel Martin et al. ===================================== src/xopen/__init__.py ===================================== @@ -732,10 +732,20 @@ def xopen( detected_format = _detect_format_from_content(filename) if detected_format == "gz": - return _open_gz(filename, mode, compresslevel, threads) + opened_file = _open_gz(filename, mode, compresslevel, threads) elif detected_format == "xz": - return _open_xz(filename, mode) + opened_file = _open_xz(filename, mode) elif detected_format == "bz2": - return _open_bz2(filename, mode, threads) + opened_file = _open_bz2(filename, mode, threads) else: - return open(filename, mode) + opened_file = open(filename, mode) + + # The "write" method for GzipFile is very costly. Lots of python calls are + # made. To a lesser extent this is true for LzmaFile and BZ2File. By + # putting a buffer in between, the expensive write method is called much + # less. The effect is very noticeable when writing small units such as + # lines or FASTQ records. + if (isinstance(opened_file, (gzip.GzipFile, bz2.BZ2File, lzma.LZMAFile)) + and "w" in mode): + opened_file = io.BufferedWriter(opened_file) # type: ignore + return opened_file ===================================== src/xopen/_version.py ===================================== @@ -1,5 +1,5 @@ # coding: utf-8 # file generated by setuptools_scm # don't change, don't track in version control -version = '1.2.0' -version_tuple = (1, 2, 0) +version = '1.2.1' +version_tuple = (1, 2, 1) ===================================== tests/test_xopen.py ===================================== @@ -532,14 +532,16 @@ def test_write_no_threads(tmpdir, ext): klass = klasses[ext] path = str(tmpdir.join(f"out.{ext}")) with xopen(path, "wb", threads=0) as f: - assert isinstance(f, klass), f + assert isinstance(f, io.BufferedWriter) + if ext: + assert isinstance(f.raw, klass), f def test_write_gzip_no_threads_no_isal(tmpdir, xopen_without_igzip): import gzip path = str(tmpdir.join("out.gz")) with xopen_without_igzip(path, "wb", threads=0) as f: - assert isinstance(f, gzip.GzipFile), f + assert isinstance(f.raw, gzip.GzipFile), f def test_write_stdout(): @@ -630,8 +632,11 @@ def test_xopen_falls_back_to_bzip2_open(lacking_pbzip2_permissions): def test_open_many_writers(tmp_path, ext): files = [] - for i in range(1, 61): - path = tmp_path / f"{i:03d}.txt.{ext}" + # Because lzma.open allocates a lot of memory, + # open fewer files to avoid MemoryError on 32-bit architectures + n = 21 if ext == ".xz" else 61 + for i in range(1, n): + path = tmp_path / f"{i:03d}.txt{ext}" f = xopen(path, "wb", threads=2) f.write(b"hello") files.append(f) View it on GitLab: https://salsa.debian.org/med-team/python-xopen/-/commit/01d726c26b8f6bc2de4c2164a9abe978939eda8b -- View it on GitLab: https://salsa.debian.org/med-team/python-xopen/-/commit/01d726c26b8f6bc2de4c2164a9abe978939eda8b You're receiving this email because of your account on salsa.debian.org.
_______________________________________________ debian-med-commit mailing list [email protected] https://alioth-lists.debian.net/cgi-bin/mailman/listinfo/debian-med-commit
