bin/get-bugzilla-attachments-by-mimetype | 3 ++- bin/get-forum-attachments.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-)
New commits: commit 7716a43e76fe26ba31393b31cfa967f5fcb6e747 Author: Thorsten Behrens <thorsten.behr...@allotropia.de> AuthorDate: Sat Aug 6 00:48:16 2022 +0200 Commit: Thorsten Behrens <thorsten.behr...@allotropia.de> CommitDate: Sat Aug 6 00:53:47 2022 +0200 Make parallelism configurable for BZ attachment fetchers Both forum and BZ document fetchers use a hardcoded size for their threadpools. Use the habitual env var PARALLELISM to override that optionally. Change-Id: I374a33ab0cfe481e0940a1f4d5d9a53ae647f135 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/137888 Tested-by: Thorsten Behrens <thorsten.behr...@allotropia.de> Reviewed-by: Thorsten Behrens <thorsten.behr...@allotropia.de> diff --git a/bin/get-bugzilla-attachments-by-mimetype b/bin/get-bugzilla-attachments-by-mimetype index 349f428f73bc..6ce2a82d30eb 100755 --- a/bin/get-bugzilla-attachments-by-mimetype +++ b/bin/get-bugzilla-attachments-by-mimetype @@ -415,7 +415,8 @@ def generate_multi_threading(): jobs.join() -max_threads = 20 # Number of threads to create, (1 = without multi-threading) +# Number of threads to create, (1 = without multi-threading, default = 20) +max_threads = int(os.environ.get('PARALLELISM', 20)) jobs = queue.Queue() generate_multi_threading() diff --git a/bin/get-forum-attachments.py b/bin/get-forum-attachments.py index 18e9259d2133..92d30ccb183a 100755 --- a/bin/get-forum-attachments.py +++ b/bin/get-forum-attachments.py @@ -160,8 +160,8 @@ if __name__ == '__main__': sys.exit(1) processes = [] - # 10 at a time seems to work fine - with ThreadPoolExecutor(max_workers=10) as executor: + # by default, 10 at a time seems to work fine + with ThreadPoolExecutor(max_workers=int(os.environ.get('PARALLELISM', 10))) as executor: for lang, config in forums.items(): processes.append(executor.submit(get_attachments_from_url, lang, config, pathes))