Xqt has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1300091?usp=email )
Change subject: Introduce SevenZipFile class for open_archive()
......................................................................
Introduce SevenZipFile class for open_archive()
Replace the raw stdout pipe returned for 7z archives with a
RawIOBase-derived wrapper that encapsulates the underlying
subprocess, ensures proper cleanup when closing the stream,
and supports rewinding the stream.
Bug: T428322
Change-Id: I3debd03bba908103d54e8326f235f6e1913ce287
---
M pywikibot/config.py
M pywikibot/tools/__init__.py
M tests/tools_tests.py
3 files changed, 275 insertions(+), 31 deletions(-)
Approvals:
Xqt: Verified; Looks good to me, approved
diff --git a/pywikibot/config.py b/pywikibot/config.py
index 743e446..c44375e 100644
--- a/pywikibot/config.py
+++ b/pywikibot/config.py
@@ -36,6 +36,8 @@
.. version-changed:: 8.0
Editor settings has been revised. *editor* variable is None by
default. Editor detection functions were moved to :mod:`editor`.
+.. version-added:: 11.4
+ The 7-zip executable variable *cmd_7zip* was added.
"""
from __future__ import annotations
@@ -867,6 +869,14 @@
# Version 5 was added with Python 3.8. It is the default since Pywikibot 11.
pickle_protocol = 5
+# 7-zip executable setting
+# The default for 7-zip executable is '7za' but my also be '7Z'. If The
+# executable is not included in the local PATH environment variable the
+# relative or absolute path can also be given with this cmd line setting
+# like:
+# cmd_7zip = r'C:\Program Files\7-Zip\7z.exe'
+cmd_7zip = '7za'
+
# ============================
# End of configuration section
# ============================
diff --git a/pywikibot/tools/__init__.py b/pywikibot/tools/__init__.py
index 5637d18..940cf96 100644
--- a/pywikibot/tools/__init__.py
+++ b/pywikibot/tools/__init__.py
@@ -9,9 +9,11 @@
import abc
import hashlib
import importlib.metadata
+import io
import ipaddress
import os
import re
+import shutil
import stat
import subprocess
import sys
@@ -19,7 +21,7 @@
from contextlib import suppress
from functools import total_ordering, wraps
from types import TracebackType
-from typing import Any
+from typing import IO, Any, Literal
from warnings import catch_warnings, showwarning, warn
import packaging.version
@@ -81,6 +83,7 @@
'strtobool',
'normalize_username',
'MediaWikiVersion',
+ 'SevenZipFile',
'open_archive',
'merge_unique_dicts',
'file_mode_checker',
@@ -540,7 +543,194 @@
return self._dev_version < other._dev_version
-def open_archive(filename: str, mode: str = 'rb', use_extension: bool = True):
+class SevenZipFile(io.RawIOBase):
+
+ """Read-only file-like wrapper around a 7za/7z subprocess.
+
+ This wrapper waits for the 7-Zip process to terminate when closing.
+ It inherits from RawIOBase and implements its low-level access
+ design. It also provides an interface similar to :class:`io.FileIO`.
+
+ This class is used by :func:`open_archive` but can also be used
+ standalone to open 7zip archives:
+
+ >>> zf = SevenZipFile('tests/data/xml/article-pyrus.xml.7z')
+ ... # doctest: +SKIP
+ >>> content = zf.readline() # doctest: +SKIP
+ >>> content[:43] # doctest: +SKIP
+ b'<mediawiki xmlns="https://www.mediawiki.org'
+ >>> zf.read(43) # doctest: +SKIP
+ b'<mediawiki xmlns="https://www.mediawiki.org'
+ >>> zf.write(b'') # doctest: +SKIP
+ Traceback (most recent call last):
+ ...
+ io.UnsupportedOperation: File or stream is not writable.
+ >>> zf.close() # doctest: +SKIP
+ >>> zf.readlines() # doctest: +SKIP
+ Traceback (most recent call last):
+ ...
+ ValueError: I/O operation on closed file.
+
+ You can use this class as context manager too:
+
+ .. code::
+
+ with SevenZipFile('tests/data/xml/article-pyrus.xml.7z') as zf:
+ content = zf.readall()
+
+ This is equal to:
+
+ .. code::
+
+ with open_archive('tests/data/xml/article-pyrus.xml.7z') as zf:
+ content = zf.readall()
+
+ but it works for 7z-files only.
+
+ The 7-Zip executable is taken from :attr:`pywikibot.config.cmd_7zip`,
+ which defaults to ``'7za'``. If the executable is not available via
+ the local ``PATH`` environment variable, a relative or absolute path
+ may be given there.
+
+ .. version-added:: 11.4
+ """
+
+ def __init__(self, name: str, /) -> None:
+ """Initializer."""
+ self._name = name
+ self._process = None
+ self._stream = None
+ self._open_process()
+
+ def __repr__(self) -> str:
+ """Representation string."""
+ module_name = type(self).__module__.removeprefix('pywikibot.')
+ class_name = f'{module_name}.{type(self).__qualname__}'
+ if self.closed:
+ return f'{class_name}[closed]'
+ return f'{class_name}({self.name!r})'
+
+ @property
+ def name(self) -> str:
+ """The file name passed to initializer."""
+ return self._name
+
+ @property
+ def mode(self) -> str:
+ """The file mode, always 'rb'."""
+ return 'rb'
+
+ def readable(self) -> Literal[True]:
+ """Return True if the stream can be read from."""
+ return True
+
+ def readinto(self, b) -> int | None:
+ """Read bytes into a pre-allocated bytes-like object *b*.
+
+ Returns an int representing the number of bytes read (0 for EOF),
+ or None if the object is set not to block and has no data to
+ read. This is the :class:`io.RawIOBase` implementation of the
+ abstract method.
+
+ :param b: Writable buffer to fill with data.
+ :return: Number of bytes read, 0 for end of file, or None if the
+ object is in non-blocking mode and no data is available.
+ :raises ValueError: I/O operation on closed file.
+ """
+ self._checkClosed()
+ self._checkReadable()
+ return self._stream.readinto(b)
+
+ def write(self, b) -> int:
+ """Write the given buffer *b* to the IO stream.
+
+ Returns the number of bytes written, which may be less than the
+ length of *b* in bytes. This is the :class:`io.RawIOBase`
+ implementation of the abstract method.
+
+ :param b: Bytes to write.
+ :raises ValueError: I/O operation on closed file.
+ :raises io.UnsupportedOperation: The stream is not writable.
+ """
+ self._checkClosed()
+ self._checkWritable()
+
+ def close(self) -> None:
+ """Close the file-like wrapper.
+
+ A closed IO object cannot be used for further I/O operations.
+ :meth:`close` may be called more than once without error.
+ """
+ if self.closed:
+ return
+
+ self._close_process()
+ super().close()
+
+ def _open_process(self) -> None:
+ """Start the 7-Zip process and initialize the stream.
+
+ :raises FileNotFoundError: The archive file or the 7-Zip executable
+ was not found.
+ :raises OSError: 7-Zip returned an error while opening the archive.
+ """
+ if not os.path.exists(self.name):
+ raise FileNotFoundError(f'Compressed file {self.name!r} not found')
+
+ cmd = shutil.which(pywikibot.config.cmd_7zip)
+ if cmd is None:
+ raise FileNotFoundError(
+ '7-Zip executable not found (not installed or not in PATH)')
+
+ self._process = subprocess.Popen(
+ [cmd, 'e', '-bd', '-so', self.name],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ bufsize=65535
+ )
+
+ stderr = self._process.stderr.read()
+ self._process.stderr.close()
+
+ if stderr:
+ self._close_process()
+ raise OSError(
+ f'Unexpected STDERR output from {cmd}:\n{stderr}')
+
+ self._stream = self._process.stdout
+
+ def _close_process(self) -> None:
+ """Close the stream and wait for the 7-Zip process to terminate."""
+ if self._stream is not None:
+ self._stream.close()
+
+ if self._process is not None:
+ self._process.wait()
+
+ self._stream = None
+ self._process = None
+
+ def rewind(self) -> None:
+ """Rewind the stream to the beginning.
+
+ This method closes the current stream, waits for the associated
+ 7-Zip process to terminate and starts a new process to read the
+ archive again from the beginning.
+
+ :raises ValueError: I/O operation on closed file.
+ :raises FileNotFoundError: The archive file or the 7-Zip executable
+ was not found.
+ :raises OSError: 7-Zip returned an error while reopening the archive.
+ """
+ self._checkClosed()
+ self._close_process()
+ self._open_process()
+
+
+@deprecated_signature(since='11.4.0')
+def open_archive(filename: str, /,
+ mode: str = 'rb', *,
+ use_extension: bool = True) -> IO[bytes]:
"""Open a file and uncompress it if needed.
This function supports bzip2, gzip, 7zip, lzma, and xz as
@@ -553,6 +743,9 @@
ending.
.. version-added:: 3.0
+ .. version-changed:: 11.4
+ *filename* parameter is positional only, *use_extension* is
+ keyword only. Uses :class:`SevenZipFile` to open 7zip-files.
:param filename: The filename.
:param mode: The mode in which the file should be opened. It may
@@ -570,11 +763,11 @@
is 7z. It is also raised by bz2 when its content is invalid.
gzip does not immediately raise that error but only on reading
it.
+ :raises NotImplementedError: When trying to write a 7z file.
:raises lzma.LZMAError: When error occurs during compression or
decompression or when initializing the state with lzma or xz.
:return: A file-like object returning the uncompressed data in
binary mode.
- :rtype: file-like object
"""
# extension_map maps magic_number to extension.
# Unfortunately, legacy LZMA container has no magic number
@@ -616,23 +809,7 @@
elif extension == '7z':
if mode != 'rb':
raise NotImplementedError('It is not possible to write a 7z file.')
-
- try:
- process = subprocess.Popen(['7za', 'e', '-bd', '-so', filename],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- bufsize=65535)
- except OSError:
- raise ValueError(
- f'7za is not installed or cannot uncompress "{filename}"')
-
- stderr = process.stderr.read()
- process.stderr.close()
- if stderr != b'':
- process.stdout.close()
- raise OSError(
- f'Unexpected STDERR output from 7za {stderr}')
- binary = process.stdout
+ binary = SevenZipFile(filename)
elif extension in ('lzma', 'xz'):
lzma_fmts = {'lzma': lzma.FORMAT_ALONE, 'xz': lzma.FORMAT_XZ}
diff --git a/tests/tools_tests.py b/tests/tools_tests.py
index 7ed8bb5..4b39979 100755
--- a/tests/tools_tests.py
+++ b/tests/tools_tests.py
@@ -8,8 +8,8 @@
import decimal
import hashlib
+import io
import os
-import subprocess
import tempfile
import unittest
from collections import Counter, OrderedDict
@@ -20,6 +20,7 @@
from pywikibot import config, tools
from pywikibot.tools import (
+ SevenZipFile,
cached,
classproperty,
has_module,
@@ -83,17 +84,73 @@
self._get_content(self.base_file + '.gz'), self.original_content)
def test_open_archive_7z(self) -> None:
- """Test open_archive with 7za if installed."""
- with skipping(OSError, msg='7za not installed'):
- subprocess.Popen(['7za'], stdout=subprocess.PIPE).stdout.close()
-
- self.assertEqual(
- self._get_content(self.base_file + '.7z'), self.original_content)
+ """Test open_archive with 7z if installed."""
+ with skipping(FileNotFoundError):
+ self.assertEqual(
+ self._get_content(self.base_file + '.7z'),
+ self.original_content
+ )
with self.assertRaisesRegex(
OSError,
- 'Unexpected STDERR output from 7za '):
- self._get_content(self.base_file + '_invalid.7z',
- use_extension=True)
+ 'Unexpected STDERR output from .*7z'):
+ self._get_content(self.base_file + '_invalid.7z')
+
+ def test_7_zip_file(self) -> None:
+ """Test SevenZipFile class."""
+ filename = self.base_file + '.7z'
+ with skipping(FileNotFoundError), tools.open_archive(filename) as zf:
+ self.assertIsInstance(zf, SevenZipFile)
+ self.assertIsSubclass(type(zf), io.RawIOBase)
+ self.assertEqual(zf.name, filename)
+ self.assertEqual(zf.mode, 'rb')
+ self.assertFalse(zf.closed)
+ self.assertTrue(zf.readable())
+ self.assertEqual(repr(zf), f'tools.SevenZipFile({filename!r})')
+
+ for method_name in ('isatty', 'seekable', 'writable'):
+ with self.subTest(method=method_name):
+ method = getattr(zf, method_name)
+ self.assertFalse(method())
+
+ for method_name in ('fileno', 'tell', 'truncate'):
+ with (
+ self.subTest(method=method_name),
+ self.assertRaises(io.UnsupportedOperation)
+ ):
+ method = getattr(zf, method_name)
+ method()
+
+ with self.assertRaises(io.UnsupportedOperation):
+ zf.seek(0)
+
+ with self.assertRaises(io.UnsupportedOperation):
+ zf.write(b'RawIOBase.write method is not implemented')
+ with self.assertRaises(io.UnsupportedOperation):
+ zf.writelines([b'foo\n', b'bar\n'])
+
+ self.assertEqual(zf.read(100), self.original_content[:100])
+ zf.rewind()
+ buffer = zf.read(-1)
+ zf.rewind()
+ self.assertEqual(buffer, zf.readall())
+ zf.rewind()
+
+ lines = self.original_content.splitlines(keepends=True)
+ self.assertEqual(zf.readline(), lines[0])
+ zf.rewind()
+ self.assertEqual(zf.readlines(), lines)
+
+ self.assertTrue(zf.closed)
+ self.assertIsNone(zf._stream)
+ self.assertIsNone(zf._process)
+ self.assertEqual(repr(zf), 'tools.SevenZipFile[closed]')
+ with self.assertRaisesRegex(ValueError,
+ r'I/O operation on closed file\.'):
+ zf.read(100)
+ with self.assertRaisesRegex(ValueError,
+ r'I/O operation on closed file\.'):
+ zf.rewind()
+ zf.close() # test that nothing happens here
def test_open_archive_lzma(self) -> None:
"""Test open_archive with lzma compressor in the standard library."""
@@ -153,7 +210,7 @@
ValueError,
'Magic number detection only when reading'):
tools.open_archive('/dev/null', # writing without extension
- 'wb', False)
+ 'wb', use_extension=False)
def test_binary_mode(self) -> None:
"""Test that it uses binary mode."""
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1300091?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I3debd03bba908103d54e8326f235f6e1913ce287
Gerrit-Change-Number: 1300091
Gerrit-PatchSet: 4
Gerrit-Owner: Xqt <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
