jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1337394?usp=email )
Change subject: cache: Stream cache directory entries with os.scandir()
......................................................................
cache: Stream cache directory entries with os.scandir()
Process cache paths as they are discovered instead of building a full
list before processing. This reduces directory-listing memory use and
allows processing to start before the scan finishes.
Materialize paths only for random sampling and close the directory
iterator through a context manager. Preserve single-file input and
fresh access-time checks. Exercise both streaming and sampling in the
existing cache test.
Change-Id: Ia2cbd9ed45b32e73ca2bbf129f20b2792283c324
---
M scripts/maintenance/cache.py
M tests/cache_tests.py
2 files changed, 70 insertions(+), 64 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/maintenance/cache.py b/scripts/maintenance/cache.py
index c623642..e85bc75 100755
--- a/scripts/maintenance/cache.py
+++ b/scripts/maintenance/cache.py
@@ -74,6 +74,7 @@
import os
import pickle
import sys
+from contextlib import nullcontext
from pathlib import Path
from random import sample
@@ -244,74 +245,77 @@
return
if os.path.isdir(cache_path):
- filenames = [os.path.join(cache_path, filename)
- for filename in os.listdir(cache_path)]
+ context = os.scandir(cache_path)
else:
- filenames = [cache_path]
+ context = nullcontext([cache_path])
- if tests:
- filenames = sample(filenames, min(len(filenames), tests))
+ with context as files:
+ filenames = (os.fspath(file) for file in files)
+ if tests:
+ filenames = list(filenames)
+ filenames = sample(filenames, min(len(filenames), tests))
- for filepath in filenames:
- filename = os.path.basename(filepath)
- cache_dir = os.path.dirname(filepath)
- if use_accesstime is not False:
- stinfo = os.stat(filepath)
+ for filepath in filenames:
+ filename = os.path.basename(filepath)
+ cache_dir = os.path.dirname(filepath)
+ if use_accesstime is not False:
+ stinfo = os.stat(filepath)
- entry = CacheEntry(cache_dir, filename)
+ entry = CacheEntry(cache_dir, filename)
- # Deletion is chosen only, abbreviate this request
- if func is None and output_func is None \
- and action_func == CacheEntry._delete:
- action_func(entry)
- continue
-
- # Skip foreign python specific directory
- *_, version = cache_path.partition('-')
- if version and version[-1] != str(PYTHON_VERSION[0]):
- pywikibot.error(f"Skipping {cache_path} directory, can't read "
- f'content with python {PYTHON_VERSION[0]}')
- continue
-
- try:
- entry._load_cache()
- except ValueError:
- pywikibot.error(f'Failed loading {entry._cachefile_path()}')
- pywikibot.exception()
- continue
-
- if use_accesstime is None:
- stinfo2 = os.stat(filepath)
- use_accesstime = stinfo.st_atime != stinfo2.st_atime
-
- if use_accesstime:
- # Reset access times to values before loading cache entry.
- os.utime(filepath, (stinfo.st_atime, stinfo.st_mtime))
- entry.stinfo = stinfo
-
- try:
- entry.parse_key()
- except ParseError as e:
- pywikibot.error(
- f'Problems parsing {entry.filename} with key {entry.key}')
- pywikibot.error(e)
- continue
-
- try:
- entry._rebuild()
- except Exception:
- pywikibot.error(f'Problems loading {entry.filename} with key '
- f'{entry.key}, {entry._parsed_key!r}')
- pywikibot.exception()
- continue
-
- if func is None or func(entry):
- if output_func or action_func is None:
- output = entry if output_func is None else output_func(entry)
- if output is not None:
- pywikibot.info(output)
- if action_func:
+ # Deletion is chosen only, abbreviate this request
+ if func is None and output_func is None \
+ and action_func == CacheEntry._delete:
action_func(entry)
+ continue
+
+ # Skip foreign python specific directory
+ *_, version = cache_path.partition('-')
+ if version and version[-1] != str(PYTHON_VERSION[0]):
+ pywikibot.error(f"Skipping {cache_path} directory, can't read "
+ f'content with python {PYTHON_VERSION[0]}')
+ continue
+
+ try:
+ entry._load_cache()
+ except ValueError:
+ pywikibot.error(f'Failed loading {entry._cachefile_path()}')
+ pywikibot.exception()
+ continue
+
+ if use_accesstime is None:
+ stinfo2 = os.stat(filepath)
+ use_accesstime = stinfo.st_atime != stinfo2.st_atime
+
+ if use_accesstime:
+ # Reset access times to values before loading cache entry.
+ os.utime(filepath, (stinfo.st_atime, stinfo.st_mtime))
+ entry.stinfo = stinfo
+
+ try:
+ entry.parse_key()
+ except ParseError as e:
+ pywikibot.error(
+ f'Problems parsing {entry.filename} with key {entry.key}')
+ pywikibot.error(e)
+ continue
+
+ try:
+ entry._rebuild()
+ except Exception:
+ pywikibot.error(f'Problems loading {entry.filename} with key '
+ f'{entry.key}, {entry._parsed_key!r}')
+ pywikibot.exception()
+ continue
+
+ if func is None or func(entry):
+ if output_func or action_func is None:
+ output = (entry if output_func is None
+ else output_func(entry))
+ if output is not None:
+ pywikibot.info(output)
+ if action_func:
+ action_func(entry)
def _parse_command(command, name):
diff --git a/tests/cache_tests.py b/tests/cache_tests.py
index e34dd63..de0f698 100755
--- a/tests/cache_tests.py
+++ b/tests/cache_tests.py
@@ -41,8 +41,10 @@
def test_cache(self) -> None:
"""Test the apicache by doing _check_cache_entry over each entry."""
with patch.object(cache.APISite, 'login', return_value=None):
- cache.process_entries(join_cache_path(), self._check_cache_entry,
- tests=25)
+ for tests in (None, 25):
+ with self.subTest(tests=tests):
+ cache.process_entries(join_cache_path(),
+ self._check_cache_entry, tests=tests)
if __name__ == '__main__':
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1337394?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ia2cbd9ed45b32e73ca2bbf129f20b2792283c324
Gerrit-Change-Number: 1337394
Gerrit-PatchSet: 2
Gerrit-Owner: Mahveotm <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
