[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/, lib/portage/util/_eventloop/
commit: 3cc986f87ddda86ee93770e03cca06346aee54c5
Author: Zac Medico gentoo org>
AuthorDate: Fri Feb 23 06:06:14 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Feb 23 06:48:29 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3cc986f8
AsyncioEventLoop: Call process.run_exitfuncs() before close
For the event loop running in the main thread, call
process.run_exitfuncs() before close with the event loop
running so that anything attached can clean itself up (like
the socks5 ProxyManager for bug 925240). This is necessary
because process.spawn uses the event loop to implement the
new returnproc parameter related to bug 916566.
Bug: https://bugs.gentoo.org/916566
Bug: https://bugs.gentoo.org/925240
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/test_socks5.py | 51 ++-
lib/portage/util/_eventloop/asyncio_event_loop.py | 44 +++
lib/portage/util/socks5.py| 16 ++-
3 files changed, 101 insertions(+), 10 deletions(-)
diff --git a/lib/portage/tests/util/test_socks5.py
b/lib/portage/tests/util/test_socks5.py
index 7b33cb3f6b..4a6d08169d 100644
--- a/lib/portage/tests/util/test_socks5.py
+++ b/lib/portage/tests/util/test_socks5.py
@@ -12,6 +12,8 @@ import time
import portage
from portage.tests import TestCase
from portage.util import socks5
+from portage.util.futures.executor.fork import ForkExecutor
+from portage.util._eventloop.global_event_loop import global_event_loop
from portage.const import PORTAGE_BIN_PATH
from http.server import BaseHTTPRequestHandler, HTTPServer
@@ -189,8 +191,10 @@ class Socks5ServerTestCase(TestCase):
path = "/index.html"
proxy = None
tempdir = tempfile.mkdtemp()
+previous_exithandlers = portage.process._exithandlers
try:
+portage.process._exithandlers = []
with AsyncHTTPServer(host, {path: content}, loop) as server:
settings = {
"PORTAGE_TMPDIR": tempdir,
@@ -211,5 +215,50 @@ class Socks5ServerTestCase(TestCase):
self.assertEqual(result, content)
finally:
-await socks5.proxy.stop()
+try:
+# Also run_exitfuncs to test atexit hook cleanup.
+await socks5.proxy.stop()
+self.assertNotEqual(portage.process._exithandlers, [])
+portage.process.run_exitfuncs()
+self.assertEqual(portage.process._exithandlers, [])
+finally:
+portage.process._exithandlers = previous_exithandlers
+shutil.rmtree(tempdir)
+
+
+class Socks5ServerLoopCloseTestCase(TestCase):
+"""
+For bug 925240, test that the socks5 proxy is automatically
+terminated when the main event loop is closed, using a subprocess
+for isolation.
+"""
+
+def testSocks5ServerLoopClose(self):
+asyncio.run(self._testSocks5ServerLoopClose())
+
+async def _testSocks5ServerLoopClose(self):
+loop = asyncio.get_running_loop()
+self.assertEqual(
+await loop.run_in_executor(
+ForkExecutor(loop=loop),
self._testSocks5ServerLoopCloseSubprocess
+),
+True,
+)
+
+@staticmethod
+def _testSocks5ServerLoopCloseSubprocess():
+loop = global_event_loop()
+tempdir = tempfile.mkdtemp()
+try:
+settings = {
+"PORTAGE_TMPDIR": tempdir,
+"PORTAGE_BIN_PATH": PORTAGE_BIN_PATH,
+}
+
+socks5.get_socks5_proxy(settings)
+loop.run_until_complete(socks5.proxy.ready())
+finally:
+loop.close()
shutil.rmtree(tempdir)
+
+return not socks5.proxy.is_running()
diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py
b/lib/portage/util/_eventloop/asyncio_event_loop.py
index b9e96bb20e..ee9e4c60ef 100644
--- a/lib/portage/util/_eventloop/asyncio_event_loop.py
+++ b/lib/portage/util/_eventloop/asyncio_event_loop.py
@@ -1,8 +1,9 @@
-# Copyright 2018-2023 Gentoo Authors
+# Copyright 2018-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import os
import signal
+import threading
import asyncio as _real_asyncio
from asyncio.events import AbstractEventLoop as _AbstractEventLoop
@@ -14,6 +15,7 @@ except ImportError:
PidfdChildWatcher = None
import portage
+from portage.util import socks5
class AsyncioEventLoop(_AbstractEventLoop):
@@ -25,18 +27,14 @@ class AsyncioEventLoop(_AbstractEventLoop):
def __init__(self, loop=None):
loop = loop or _real_asyncio.get_event_loop()
self._loop = loop
-self.run_until_complete = (
-self._run_until_complete
-if portage._internal_caller
-else loop.run_until_complete
-)
+self.run_until_complete = self._run_until_complete
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/, lib/portage/tests/bin/, ...
commit: d88ee7e03f900a16904f3c428449d71710ecaebd
Author: Sam James gentoo org>
AuthorDate: Tue Jan 10 13:21:13 2023 +
Commit: Sam James gentoo org>
CommitDate: Tue Jan 10 15:12:38 2023 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d88ee7e0
*/*: run Black post-flynt
Signed-off-by: Sam James gentoo.org>
bin/binhost-snapshot | 3 +-
bin/egencache | 24 ++--
bin/glsa-check | 16 ++
bin/portageq | 8 +--
bin/quickpkg | 3 +-
bin/regenworld | 4 +-
lib/_emerge/AbstractEbuildProcess.py | 4 +-
lib/_emerge/AsynchronousTask.py| 4 +-
lib/_emerge/BlockerDB.py | 4 +-
lib/_emerge/EbuildPhase.py | 6 +-
lib/_emerge/Package.py | 2 +-
lib/_emerge/PackageUninstall.py| 4 +-
lib/_emerge/Scheduler.py | 6 +-
lib/_emerge/UseFlagDisplay.py | 4 +-
lib/_emerge/actions.py | 30 +++---
lib/_emerge/depgraph.py| 64 ++
lib/_emerge/resolver/circular_dependency.py| 10 +---
lib/_emerge/resolver/output.py | 4 +-
lib/_emerge/resolver/output_helpers.py | 4 +-
lib/_emerge/resolver/slot_collision.py | 4 +-
lib/_emerge/search.py | 5 +-
lib/portage/_emirrordist/DeletionTask.py | 12 +---
lib/portage/_emirrordist/FetchIterator.py | 4 +-
lib/portage/_emirrordist/FetchTask.py | 4 +-
lib/portage/_sets/files.py | 9 +--
lib/portage/cache/sqlite.py| 4 +-
lib/portage/cache/template.py | 3 +-
lib/portage/checksum.py| 4 +-
lib/portage/dbapi/porttree.py | 2 +-
lib/portage/dbapi/vartree.py | 22 ++--
lib/portage/dep/__init__.py| 3 +-
lib/portage/dep/dep_check.py | 6 +-
lib/portage/emaint/main.py | 4 +-
lib/portage/emaint/modules/move/move.py| 4 +-
lib/portage/emaint/modules/resume/resume.py| 4 +-
lib/portage/emaint/modules/world/world.py | 4 +-
lib/portage/locks.py | 4 +-
lib/portage/package/ebuild/config.py | 4 +-
lib/portage/package/ebuild/doebuild.py | 4 +-
lib/portage/package/ebuild/fetch.py| 8 +--
lib/portage/process.py | 4 +-
lib/portage/sync/modules/git/git.py| 4 +-
lib/portage/sync/modules/mercurial/mercurial.py| 4 +-
lib/portage/sync/modules/rsync/rsync.py| 8 +--
lib/portage/sync/modules/svn/svn.py| 6 +-
lib/portage/sync/modules/webrsync/webrsync.py | 4 +-
lib/portage/sync/syncbase.py | 4 +-
lib/portage/tests/bin/test_eapi7_ver_funcs.py | 16 ++
lib/portage/tests/dep/testAtom.py | 8 +--
lib/portage/tests/ebuild/test_fetch.py | 4 +-
lib/portage/tests/process/test_unshare_net.py | 4 +-
lib/portage/tests/resolver/ResolverPlayground.py | 10 ++--
lib/portage/tests/sync/test_sync_local.py | 4 +-
lib/portage/tests/util/test_install_mask.py| 4 +-
lib/portage/update.py | 4 +-
lib/portage/util/__init__.py | 4 +-
lib/portage/util/_dyn_libs/LinkageMapELF.py| 4 +-
.../util/_dyn_libs/display_preserved_libs.py | 5 +-
lib/portage/util/digraph.py| 4 +-
lib/portage/util/futures/executor/fork.py | 2 +-
lib/portage/util/movefile.py | 8 +--
lib/portage/versions.py| 3 +-
runtests | 4 +-
setup.py | 2 +-
64 files changed, 122 insertions(+), 324 deletions(-)
diff --git a/bin/binhost-snapshot b/bin/binhost-snapshot
index 5fac0b46c..05d2f514b 100755
--- a/bin/binhost-snapshot
+++ b/bin/binhost-snapshot
@@ -82,8 +82,7 @@ def main(argv):
if not os.path.isfile(src_pkgs_index):
parser.error(
-"src_pkg_dir does not contain a "
-+ f"'Packages' index: '{src_pkg_dir}'"
+"src_pkg_dir does not contain a " + f"'Packages' index:
'{src_pkg_dir}'"
)
parse_result = urlparse(snapshot_uri)
diff --git a/bin/egencache b/bin/egencache
index 090b9b1ed..0050ca32a 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -254,9 +254,7 @@ try:
load_average = 0.0
if
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/
commit: 3a349232febc295565dee8dd1b25f9a76726b50a Author: Michał Górny gentoo org> AuthorDate: Fri Dec 30 10:37:03 2022 + Commit: Sam James gentoo org> CommitDate: Sat Dec 31 13:33:03 2022 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3a349232 Inline Whirlpool tests in test_whirlpool Inline the Whirlpool tests in the test case instead of spawning them as a subprocess. Signed-off-by: Michał Górny gentoo.org> Signed-off-by: Sam James gentoo.org> lib/portage/tests/util/test_whirlpool.py | 34 ++-- lib/portage/util/whirlpool.py| 29 +++ 2 files changed, 22 insertions(+), 41 deletions(-) diff --git a/lib/portage/tests/util/test_whirlpool.py b/lib/portage/tests/util/test_whirlpool.py index 554fffc6a..d93467b21 100644 --- a/lib/portage/tests/util/test_whirlpool.py +++ b/lib/portage/tests/util/test_whirlpool.py @@ -1,23 +1,27 @@ -# Copyright 2011-2014 Gentoo Foundation +# Copyright 2011-2022 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import subprocess - -import portage -from portage import os -from portage.const import PORTAGE_PYM_PATH from portage.tests import TestCase +from portage.util.whirlpool import Whirlpool class WhirlpoolTestCase(TestCase): def testBundledWhirlpool(self): -# execute the tests bundled with the whirlpool module -retval = subprocess.call( -[ -portage._python_interpreter, -"-b", -"-Wd", -os.path.join(PORTAGE_PYM_PATH, "portage/util/whirlpool.py"), -] +self.assertEqual( +Whirlpool(b"The quick brown fox jumps over the lazy dog").hexdigest(), + "b97de512e91e3828b40d2b0fdce9ceb3c4a71f9bea8d88e75c4fa854df36725fd2b52eb6544edcacd6f8beddfea403cb55ae31f03ad62a5ef54e42ee82c3fb35", +) +self.assertEqual( +Whirlpool(b"The quick brown fox jumps over the lazy eog").hexdigest(), + "c27ba124205f72e6847f3e19834f925cc666d0974167af915bb462420ed40cc50900d85a1f923219d832357750492d5c143011a76988344c2635e69d06f2d38c", +) +self.assertEqual( +Whirlpool(b"").hexdigest(), + "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3", +) +w = Whirlpool() +w.update(b"") +self.assertEqual( +w.hexdigest(), + "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3", ) -self.assertEqual(retval, os.EX_OK) diff --git a/lib/portage/util/whirlpool.py b/lib/portage/util/whirlpool.py index b62191d91..1431c18fb 100644 --- a/lib/portage/util/whirlpool.py +++ b/lib/portage/util/whirlpool.py @@ -1,3 +1,6 @@ +# Copyright 2022 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + # whirlpool.py - pure Python implementation of the Whirlpool algorithm. # Bjorn Edstrom 16 december 2007. ## @@ -2372,29 +2375,3 @@ def processBuffer(ctx): # apply the Miyaguchi-Preneel compression function for i in range(8): ctx.hash[i] ^= state[i] ^ block[i] - - -# -# Tests. -# - - -if __name__ == "__main__": -assert ( -Whirlpool(b"The quick brown fox jumps over the lazy dog").hexdigest() -== "b97de512e91e3828b40d2b0fdce9ceb3c4a71f9bea8d88e75c4fa854df36725fd2b52eb6544edcacd6f8beddfea403cb55ae31f03ad62a5ef54e42ee82c3fb35" -) -assert ( -Whirlpool(b"The quick brown fox jumps over the lazy eog").hexdigest() -== "c27ba124205f72e6847f3e19834f925cc666d0974167af915bb462420ed40cc50900d85a1f923219d832357750492d5c143011a76988344c2635e69d06f2d38c" -) -assert ( -Whirlpool(b"").hexdigest() -== "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3" -) -w = Whirlpool() -w.update(b"") -assert ( -w.hexdigest() -== "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3" -)
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/
commit: cd6a9fa4338a2444c02ece8ff9fb59971a4fe98d
Author: David Palao gmail com>
AuthorDate: Wed May 25 13:29:54 2022 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue Jun 7 23:47:55 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cd6a9fa4
test(mtimedb): added two more UTs about MtimeDB instances creation
Signed-off-by: David Palao gmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/tests/util/test_mtimedb.py | 23 ---
lib/portage/util/mtimedb.py| 6 --
2 files changed, 24 insertions(+), 5 deletions(-)
diff --git a/lib/portage/tests/util/test_mtimedb.py
b/lib/portage/tests/util/test_mtimedb.py
index 6ec009c38..e6ddf5b80 100644
--- a/lib/portage/tests/util/test_mtimedb.py
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -126,7 +126,7 @@ _PARTIAL_FILE_JSON = b"""{
}
"""
-_TWO_RESUME_LIST_JSON = b"""{
+_TWO_RESUME_LISTS_JSON = b"""{
"info": {
"/usr/share/binutils-data/x86_64-pc-linux-gnu/2.34/info":
1711787325,
"/usr/share/gcc-data/x86_64-pc-linux-gnu/11.2.0/info":
1735158257,
@@ -213,12 +213,12 @@ _TWO_RESUME_LIST_JSON = b"""{
class MtimeDBTestCase(TestCase):
text = b"Unit tests for MtimeDB"
-def test_has_only_allowed_keys(self):
+def test_instance_created_with_only_expected_keys(self):
all_fixtures = (
_ONE_RESUME_LIST_JSON,
_EMPTY_FILE,
_PARTIAL_FILE_JSON,
-_TWO_RESUME_LIST_JSON,
+_TWO_RESUME_LISTS_JSON,
)
for contents in all_fixtures:
with patch(
@@ -226,3 +226,20 @@ class MtimeDBTestCase(TestCase):
):
mtimedb = MtimeDB("/path/to/mtimedb")
self.assertLessEqual(set(mtimedb.keys()), _MTIMEDBKEYS)
+
+def test_instance_has_default_values(self):
+with patch('portage.util.mtimedb.open',
+ mock_open(read_data=_EMPTY_FILE)):
+mtimedb = MtimeDB("/some/path/mtimedb")
+self.assertEqual(mtimedb["starttime"], 0)
+self.assertEqual(mtimedb["version"], "")
+self.assertEqual(mtimedb["info"], {})
+self.assertEqual(mtimedb["ldpath"], {})
+self.assertEqual(mtimedb["updates"], {})
+
+def test_instance_has_a_deepcopy_of_clean_data(self):
+with patch('portage.util.mtimedb.open',
+ mock_open(read_data=_ONE_RESUME_LIST_JSON)):
+mtimedb = MtimeDB("/some/path/mtimedb")
+self.assertEqual(dict(mtimedb), dict(mtimedb._clean_data))
+self.assertIsNot(mtimedb, mtimedb._clean_data)
diff --git a/lib/portage/util/mtimedb.py b/lib/portage/util/mtimedb.py
index 37afba7c8..3aab0b90b 100644
--- a/lib/portage/util/mtimedb.py
+++ b/lib/portage/util/mtimedb.py
@@ -73,7 +73,8 @@ class MtimeDB(dict):
try:
d = json.loads(
_unicode_decode(
-content, encoding=_encodings["repo.content"],
errors="strict"
+content, encoding=_encodings["repo.content"],
+errors="strict"
)
)
except SystemExit:
@@ -91,7 +92,8 @@ class MtimeDB(dict):
raise
except Exception:
writemsg(
-_(f"!!! Error loading '{filename}': {e}\n"),
noiselevel=-1
+_(f"!!! Error loading '{filename}': {e}\n"),
+noiselevel=-1
)
if "old" in d:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/
commit: f753c049d5ecd6b7d7b01b4efa678c59c49bcebc
Author: David Palao gmail com>
AuthorDate: Tue May 24 15:47:56 2022 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue Jun 7 23:47:55 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f753c049
test(mtimedb): added first unit tests for MtimeDB
Signed-off-by: David Palao gmail.com>
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/tests/util/test_mtimedb.py | 228 +
lib/portage/util/mtimedb.py| 2 +-
2 files changed, 229 insertions(+), 1 deletion(-)
diff --git a/lib/portage/tests/util/test_mtimedb.py
b/lib/portage/tests/util/test_mtimedb.py
new file mode 100644
index 0..6ec009c38
--- /dev/null
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -0,0 +1,228 @@
+# Copyright 2022 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from unittest.mock import patch, mock_open
+
+from portage.tests import TestCase
+
+from portage.util.mtimedb import MtimeDB, _MTIMEDBKEYS
+from portage.exception import DigestException
+
+
+_ONE_RESUME_LIST_JSON = b"""{
+ "info": {
+
"/tmp/stage1root/usr/share/binutils-data/x86_64-pc-linux-gnu/2.34/info":
1711785090,
+
"/tmp/stage1root/usr/share/gcc-data/x86_64-pc-linux-gnu/9.3.0/info": 1711785090,
+ "/tmp/stage1root/usr/share/info": 1611785090,
+ "/usr/share/binutils-data/x86_64-pc-linux-gnu/2.34/info":
1711787325,
+ "/usr/share/gcc-data/x86_64-pc-linux-gnu/11.2.0/info":
1735158257,
+ "/usr/share/gcc-data/x86_64-pc-linux-gnu/9.3.0/info":
1711787325,
+ "/usr/share/info": 1650633847
+ },
+ "ldpath": {
+ "/lib": 1748456830,
+ "/lib64": 1750523381,
+ "/usr/lib": 1750461195,
+ "/usr/lib/llvm/11/lib64": 1723048948,
+ "/usr/lib/llvm/12/lib64": 1730499781,
+ "/usr/lib/llvm/13/lib64": 1747003135,
+ "/usr/lib/rust/lib": 1750461173,
+ "/usr/lib64": 1750881821,
+ "/usr/local/lib": 1711784303,
+ "/usr/local/lib64": 1711784303
+ },
+ "resume": {
+ "favorites": [
+ "@world"
+ ],
+ "mergelist": [
+ [
+ "ebuild",
+ "/",
+ "some-cat/some-package-1.2.3-r4",
+ "merge"
+ ],
+ [
+ "ebuild",
+ "/",
+ "another-cat/another-package-4.3.2-r1",
+ "merge"
+ ]
+ ],
+ "myopts": {
+ "--buildpkg": true,
+ "--deep": true,
+ "--getbinpkg": true,
+ "--keep-going": true,
+ "--newuse": true,
+ "--quiet": true,
+ "--regex-search-auto": "y",
+ "--update": true,
+ "--usepkg": true,
+ "--verbose": true
+ }
+ },
+ "starttime": 0,
+ "updates": {
+ "/var/db/repos/gentoo/profiles/updates/1Q-2021": 1739992409,
+ "/var/db/repos/gentoo/profiles/updates/1Q-2022": 1747854791,
+ "/var/db/repos/gentoo/profiles/updates/2Q-2021": 1724404379,
+ "/var/db/repos/gentoo/profiles/updates/2Q-2022": 1752846209,
+ "/var/db/repos/gentoo/profiles/updates/3Q-2021": 1741119203,
+ "/var/db/repos/gentoo/profiles/updates/4Q-2020": 1709167362,
+ "/var/db/repos/gentoo/profiles/updates/4Q-2021": 1742787797
+ },
+ "version": "3.0.30"
+}
+"""
+
+_EMPTY_FILE = b""
+
+_PARTIAL_FILE_JSON = b"""{
+ "ldpath": {
+ "/lib": 1748456830,
+ "/lib64": 1750523381,
+ "/usr/lib": 1750461195,
+ "/usr/lib/llvm/11/lib64": 1723048948,
+ "/usr/lib/llvm/12/lib64": 1730499781,
+ "/usr/lib/llvm/13/lib64": 1747003135,
+ "/usr/lib/rust/lib": 1750461173,
+ "/usr/lib64": 1750881821,
+ "/usr/local/lib": 1711784303,
+ "/usr/local/lib64": 1711784303
+ },
+ "resume": {
+ "favorites": [
+ "@world"
+ ],
+ "mergelist": [
+ [
+ "ebuild",
+ "/",
+ "some-cat/some-package-1.2.3-r4",
+ "merge"
+ ],
+ [
+ "ebuild",
+
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/
commit: 5cc166489106da93c20ad3c5bb138ba8a359de2c
Author: Jeff Chase google com>
AuthorDate: Tue May 4 17:29:58 2021 +
Commit: Sam James gentoo org>
CommitDate: Fri Apr 15 04:26:22 2022 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5cc16648
install_mask: remove masked symlinks to directories (bug 678462)
os.walk() categorizes symlinks to directories as directories so they
were being ignored by INSTALL_MASK. This change calls os.scandir()
instead which efficiently provides more control.
[sam: cherry-picked from chromiumos' third_party/portage_tool repo]
(cherry picked from commit 6473079d07351dda49a906cb89d24a9a39526bf7)
Bug: https://bugs.gentoo.org/678462
Signed-off-by: Sam James gentoo.org>
Closes: https://github.com/gentoo/portage/pull/820
Signed-off-by: Sam James gentoo.org>
lib/portage/tests/util/test_install_mask.py | 33 +++--
lib/portage/util/install_mask.py| 18 ++--
2 files changed, 42 insertions(+), 9 deletions(-)
diff --git a/lib/portage/tests/util/test_install_mask.py
b/lib/portage/tests/util/test_install_mask.py
index d9558a857..0ed98b3a5 100644
--- a/lib/portage/tests/util/test_install_mask.py
+++ b/lib/portage/tests/util/test_install_mask.py
@@ -1,8 +1,11 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2022 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+import tempfile
+from portage import os
+from portage import shutil
from portage.tests import TestCase
-from portage.util.install_mask import InstallMask
+from portage.util.install_mask import InstallMask, install_mask_dir
class InstallMaskTestCase(TestCase):
@@ -166,3 +169,29 @@ class InstallMaskTestCase(TestCase):
install_mask_str, path
),
)
+
+def testSymlinkDir(self):
+"""
+Test that masked symlinks to directories are removed.
+"""
+tmp_dir = tempfile.mkdtemp()
+
+try:
+base_dir = os.path.join(tmp_dir, "foo")
+target_dir = os.path.join(tmp_dir, "foo", "bar")
+link_name = os.path.join(tmp_dir, "foo", "baz")
+
+os.mkdir(base_dir)
+os.mkdir(target_dir)
+os.symlink(target_dir, link_name)
+
+install_mask = InstallMask("/foo/")
+install_mask_dir(tmp_dir, install_mask)
+self.assertFalse(
+os.path.lexists(link_name), "failed to remove
{}".format(link_name)
+)
+self.assertFalse(
+os.path.lexists(base_dir), "failed to remove
{}".format(base_dir)
+)
+finally:
+shutil.rmtree(tmp_dir)
diff --git a/lib/portage/util/install_mask.py b/lib/portage/util/install_mask.py
index 2b65fc230..638c150ff 100644
--- a/lib/portage/util/install_mask.py
+++ b/lib/portage/util/install_mask.py
@@ -171,22 +171,26 @@ def install_mask_dir(base_dir, install_mask,
onerror=None):
dir_stack = []
# Remove masked files.
-for parent, dirs, files in os.walk(base_dir, onerror=onerror):
+todo = [base_dir]
+while todo:
+parent = todo.pop()
try:
parent = _unicode_decode(parent, errors="strict")
except UnicodeDecodeError:
continue
+
dir_stack.append(parent)
-for fname in files:
+for entry in os.scandir(parent):
try:
-fname = _unicode_decode(fname, errors="strict")
+abs_path = _unicode_decode(entry.path, errors="strict")
except UnicodeDecodeError:
continue
-abs_path = os.path.join(parent, fname)
-relative_path = abs_path[base_dir_len:]
-if install_mask.match(relative_path):
+
+if entry.is_dir(follow_symlinks=False):
+todo.append(entry.path)
+elif install_mask.match(abs_path[base_dir_len:]):
try:
-os.unlink(abs_path)
+os.unlink(entry.path)
except OSError as e:
onerror(e)
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/
commit: 0b11ac9ce296367699e1a191b87be22612b266ff Author: Zac Medico gentoo org> AuthorDate: Tue Jan 19 00:44:57 2021 + Commit: Zac Medico gentoo org> CommitDate: Tue Jan 19 00:45:17 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0b11ac9c ProxyManager: Use async and await syntax Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/util/test_socks5.py | 4 ++-- lib/portage/util/socks5.py| 9 +++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py index 44d522013..82cf93fe8 100644 --- a/lib/portage/tests/util/test_socks5.py +++ b/lib/portage/tests/util/test_socks5.py @@ -1,4 +1,4 @@ -# Copyright 2019 Gentoo Authors +# Copyright 2019-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import functools @@ -185,7 +185,7 @@ class Socks5ServerTestCase(TestCase): } proxy = socks5.get_socks5_proxy(settings) - loop.run_until_complete(socks5.proxy.ready(loop=loop)) + loop.run_until_complete(socks5.proxy.ready()) result = loop.run_until_complete(loop.run_in_executor(None, self._fetch_via_proxy, proxy, host, server.server_port, path)) diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index a76d1a741..ddf6bb4d0 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -1,5 +1,5 @@ # SOCKSv5 proxy manager for network-sandbox -# Copyright 2015-2020 Gentoo Authors +# Copyright 2015-2021 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -11,7 +11,6 @@ import portage.data from portage import _python_interpreter from portage.data import portage_gid, portage_uid, userpriv_groups from portage.process import atexit_register, spawn -from portage.util.futures.compat_coroutine import coroutine from portage.util.futures import asyncio class ProxyManager: @@ -74,9 +73,7 @@ class ProxyManager: """ return self.socket_path is not None - - @coroutine - def ready(self, loop=None): + async def ready(self, loop=None): """ Wait for the proxy socket to become ready. This method is a coroutine. """ @@ -98,7 +95,7 @@ class ProxyManager: except EnvironmentError as e: if e.errno != errno.ENOENT: raise - yield asyncio.sleep(0.2, loop=loop) + await asyncio.sleep(0.2) else: break finally:
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/_async/
commit: 7b8f57335c43054fe4008b7401d6ac2b3f710c1a
Author: Zac Medico gentoo org>
AuthorDate: Wed Nov 6 08:03:36 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Nov 6 20:05:27 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7b8f5733
FileCopier: capture exceptions
Use ForkExecutor to capture exceptions instead of showing
a full traceback. FileCopier callers will now be responsible
for displaying relevant exception messages.
Bug: https://bugs.gentoo.org/699400
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/test_file_copier.py | 44 ++
lib/portage/util/_async/FileCopier.py | 16 ++-
2 files changed, 53 insertions(+), 7 deletions(-)
diff --git a/lib/portage/tests/util/test_file_copier.py
b/lib/portage/tests/util/test_file_copier.py
new file mode 100644
index 0..01dfba494
--- /dev/null
+++ b/lib/portage/tests/util/test_file_copier.py
@@ -0,0 +1,44 @@
+# Copyright 2019 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import os
+import shutil
+import tempfile
+
+from portage.tests import TestCase
+from portage.util._async.FileCopier import FileCopier
+from portage.util._eventloop.global_event_loop import global_event_loop
+
+
+class FileCopierTestCase(TestCase):
+
+ def testFileCopier(self):
+ loop = global_event_loop()
+ tempdir = tempfile.mkdtemp()
+ try:
+
+ # regular successful copy
+ src_path = os.path.join(tempdir, 'src')
+ dest_path = os.path.join(tempdir, 'dest')
+ content = b'foo'
+ with open(src_path, 'wb') as f:
+ f.write(content)
+ copier = FileCopier(src_path=src_path,
dest_path=dest_path, scheduler=loop)
+ copier.start()
+ loop.run_until_complete(copier.async_wait())
+ self.assertEqual(copier.returncode, 0)
+ copier.future.result()
+ with open(dest_path, 'rb') as f:
+ self.assertEqual(f.read(), content)
+
+ # failure due to nonexistent src_path
+ src_path = os.path.join(tempdir, 'does-not-exist')
+ copier = FileCopier(src_path=src_path,
dest_path=dest_path, scheduler=loop)
+ copier.start()
+ loop.run_until_complete(copier.async_wait())
+ self.assertEqual(copier.returncode, 1)
+ self.assertEqual(copier.future.exception().errno,
errno.ENOENT)
+ self.assertEqual(copier.future.exception().filename,
src_path.encode('utf8'))
+ finally:
+ shutil.rmtree(tempdir)
diff --git a/lib/portage/util/_async/FileCopier.py
b/lib/portage/util/_async/FileCopier.py
index 27e5ab4c0..3a0be4b63 100644
--- a/lib/portage/util/_async/FileCopier.py
+++ b/lib/portage/util/_async/FileCopier.py
@@ -1,17 +1,19 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-from portage import os
from portage import shutil
-from portage.util._async.ForkProcess import ForkProcess
+from portage.util.futures import asyncio
+from portage.util.futures.executor.fork import ForkExecutor
+from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
-class FileCopier(ForkProcess):
+class FileCopier(AsyncTaskFuture):
"""
Asynchronously copy a file.
"""
__slots__ = ('src_path', 'dest_path')
- def _run(self):
- shutil.copy(self.src_path, self.dest_path)
- return os.EX_OK
+ def _start(self):
+ self.future =
asyncio.ensure_future(self.scheduler.run_in_executor(ForkExecutor(loop=self.scheduler),
+ shutil.copy, self.src_path, self.dest_path))
+ super(FileCopier, self)._start()
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/
commit: eb2674c1c2d84b2c9e9923e1c1666cb7a596c36c
Author: Zac Medico gentoo org>
AuthorDate: Sat Jan 19 08:39:47 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Jan 20 22:57:28 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=eb2674c1
INSTALL_MASK: index patterns anchored with leading slash (bug 675826)
For scalability, use a tree structure to index patterns that are
anchored with a leading slash.
Patterns anchored with leading slash are indexed by leading non-glob
components, making it possible to minimize the number of fnmatch
calls. For example:
/foo*/bar -> {'.': ['/foo*/bar']}
/foo/bar* -> {'foo': {'.': ['/foo/bar*']}}
/foo/bar/ -> {'foo': {'bar': {'.': ['/foo/bar/']}}}
Bug: https://bugs.gentoo.org/675826
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/test_install_mask.py | 36 +
lib/portage/util/install_mask.py| 84 ++---
2 files changed, 113 insertions(+), 7 deletions(-)
diff --git a/lib/portage/tests/util/test_install_mask.py
b/lib/portage/tests/util/test_install_mask.py
index f651eb4b7..6a29db79a 100644
--- a/lib/portage/tests/util/test_install_mask.py
+++ b/lib/portage/tests/util/test_install_mask.py
@@ -119,6 +119,42 @@ class InstallMaskTestCase(TestCase):
),
)
),
+ (
+ '/usr/share/locale '
+ '-/usr/share/locale/en* '
+ '-/usr/share/locale/kf5_all_languages '
+ '-/usr/share/locale/locale.alias',
+ (
+ (
+ 'usr/share/locale/en',
+ False,
+ ),
+ (
+ 'usr/share/locale/en_GB',
+ False,
+ ),
+ (
+
'usr/share/locale/en/kf5_all_languages',
+ False,
+ ),
+ (
+ 'usr/share/locale/locale.alias',
+ False,
+ ),
+ (
+ 'usr/share/locale/es',
+ True,
+ ),
+ (
+ 'usr/share/locale/fr',
+ True,
+ ),
+ (
+ 'usr/share/locale',
+ True,
+ ),
+ )
+ ),
)
for install_mask_str, paths in cases:
diff --git a/lib/portage/util/install_mask.py b/lib/portage/util/install_mask.py
index 32627eb05..a8c0cbda5 100644
--- a/lib/portage/util/install_mask.py
+++ b/lib/portage/util/install_mask.py
@@ -3,8 +3,11 @@
__all__ = ['install_mask_dir', 'InstallMask']
+import collections
import errno
import fnmatch
+import functools
+import operator
import sys
from portage import os, _unicode_decode
@@ -18,13 +21,82 @@ else:
_unicode = unicode
+def _defaultdict_tree():
+ return collections.defaultdict(_defaultdict_tree)
+
+
+_pattern = collections.namedtuple('_pattern', (
+ 'orig_index',
+ 'is_inclusive',
+ 'pattern',
+ 'leading_slash',
+))
+
+
class InstallMask(object):
def __init__(self, install_mask):
"""
@param install_mask: INSTALL_MASK value
@type install_mask: str
"""
- self._install_mask = install_mask.split()
+ # Patterns not anchored with leading slash
+ self._unanchored = []
+
+ # Patterns anchored with leading slash are indexed by leading
+ # non-glob components, making it possible to minimize the
+ # number of fnmatch calls. For example:
+ # /foo*/bar -> {'.': ['/foo*/bar']}
+ # /foo/bar* -> {'foo': {'.': ['/foo/bar*']}}
+ # /foo/bar/ -> {'foo': {'bar': {'.': ['/foo/bar/']}}}
+ self._anchored = _defaultdict_tree()
+ for orig_index, pattern in enumerate(install_mask.split()):
+ # if pattern starts with
