[gentoo-commits] proj/portage:master commit in: lib/portage/repository/storage/, lib/portage/tests/sync/
commit: b0c535b7310e7724d8feb0570b827d38c5d17fc3
Author: Zac Medico gentoo org>
AuthorDate: Fri Jul 31 21:40:53 2020 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Jul 31 22:00:21 2020 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b0c535b7
HardlinkRcuRepoStorage: handle removed sync-rcu-store-dir (bug 734990)
If SyncManager.pre_sync creates an empty directory where
self._latest_symlink is suppose to be (which is normal if
sync-rcu-store-dir has been removed), then we need to remove
the directory or else rename will raise IsADirectoryError
when we try to replace the directory with a symlink.
Bug: https://bugs.gentoo.org/734990
Signed-off-by: Zac Medico gentoo.org>
lib/portage/repository/storage/hardlink_rcu.py | 13 -
lib/portage/tests/sync/test_sync_local.py | 10 --
2 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/lib/portage/repository/storage/hardlink_rcu.py
b/lib/portage/repository/storage/hardlink_rcu.py
index 80cdbb0d7..bb2c8496b 100644
--- a/lib/portage/repository/storage/hardlink_rcu.py
+++ b/lib/portage/repository/storage/hardlink_rcu.py
@@ -1,4 +1,4 @@
-# Copyright 2018 Gentoo Foundation
+# Copyright 2018-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import datetime
@@ -204,6 +204,17 @@ class HardlinkRcuRepoStorage(RepoStorageInterface):
except OSError:
pass
os.symlink('snapshots/{}'.format(new_id), new_symlink)
+
+ # If SyncManager.pre_sync creates an empty directory where
+ # self._latest_symlink is suppose to be (which is normal if
+ # sync-rcu-store-dir has been removed), then we need to remove
+ # the directory or else rename will raise IsADirectoryError
+ # when we try to replace the directory with a symlink.
+ try:
+ os.rmdir(self._latest_symlink)
+ except OSError:
+ pass
+
os.rename(new_symlink, self._latest_symlink)
try:
diff --git a/lib/portage/tests/sync/test_sync_local.py
b/lib/portage/tests/sync/test_sync_local.py
index 08dd4fe57..efd61aac8 100644
--- a/lib/portage/tests/sync/test_sync_local.py
+++ b/lib/portage/tests/sync/test_sync_local.py
@@ -1,4 +1,4 @@
-# Copyright 2014-2015 Gentoo Foundation
+# Copyright 2014-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import datetime
@@ -71,6 +71,7 @@ class SyncLocalTestCase(TestCase):
distdir = os.path.join(eprefix, "distdir")
repo = settings.repositories["test_repo"]
metadata_dir = os.path.join(repo.location, "metadata")
+ rcu_store_dir = os.path.join(eprefix,
'var/repositories/test_repo_rcu_storedir')
cmds = {}
for cmd in ("emerge", "emaint"):
@@ -191,6 +192,10 @@ class SyncLocalTestCase(TestCase):
(homedir, lambda: os.mkdir(repo.user_location)),
)
+ delete_rcu_store_dir = (
+ (homedir, lambda: shutil.rmtree(rcu_store_dir)),
+ )
+
revert_rcu_layout = (
(homedir, lambda: os.rename(repo.user_location,
repo.user_location + '.bak')),
(homedir, lambda:
os.rename(os.path.realpath(repo.user_location + '.bak'), repo.user_location)),
@@ -289,7 +294,8 @@ class SyncLocalTestCase(TestCase):
for cwd, cmd in rename_repo + sync_cmds_auto_sync +
sync_cmds + \
rsync_opts_repos + rsync_opts_repos_default + \
rsync_opts_repos_default_ovr +
rsync_opts_repos_default_cancel + \
- bump_timestamp_cmds + sync_rsync_rcu +
sync_cmds + revert_rcu_layout + \
+ bump_timestamp_cmds + sync_rsync_rcu +
sync_cmds + delete_rcu_store_dir + \
+ sync_cmds + revert_rcu_layout + \
delete_repo_location + sync_cmds + sync_cmds + \
bump_timestamp_cmds + sync_cmds +
revert_rcu_layout + \
delete_sync_repo + git_repo_create +
sync_type_git + \
[gentoo-commits] proj/portage:master commit in: lib/portage/repository/storage/
commit: b587fc874ce95064139ba85552e146da957cce9e
Author: Zac Medico gentoo org>
AuthorDate: Fri Sep 28 05:32:31 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Sep 28 05:43:26 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b587fc87
HardlinkQuarantineRepoStorage: exclude distfiles and packages (bug 666554)
These paths have been excluded for a long time, and they are also
whitelisted in the top-level Manifest.files.gz:
IGNORE distfiles
IGNORE local
IGNORE lost+found
IGNORE packages
Bug: https://bugs.gentoo.org/666554
Signed-off-by: Zac Medico gentoo.org>
lib/portage/repository/storage/hardlink_quarantine.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/portage/repository/storage/hardlink_quarantine.py
b/lib/portage/repository/storage/hardlink_quarantine.py
index 7e9cf4493..3594cb1c9 100644
--- a/lib/portage/repository/storage/hardlink_quarantine.py
+++ b/lib/portage/repository/storage/hardlink_quarantine.py
@@ -60,6 +60,7 @@ class HardlinkQuarantineRepoStorage(RepoStorageInterface):
# Use rsync --link-dest to hardlink a files into
self._update_location,
# since cp -l is not portable.
yield self._check_call(['rsync', '-a', '--link-dest',
self._user_location,
+ '--exclude=/distfiles', '--exclude=/local',
'--exclude=/lost+found', '--exclude=/packages',
'--exclude',
'/{}'.format(os.path.basename(update_location)),
self._user_location + '/', update_location + '/'])
@@ -78,6 +79,7 @@ class HardlinkQuarantineRepoStorage(RepoStorageInterface):
update_location = self.current_update
self._update_location = None
yield self._check_call(['rsync', '-a', '--delete',
+ '--exclude=/distfiles', '--exclude=/local',
'--exclude=/lost+found', '--exclude=/packages',
'--exclude',
'/{}'.format(os.path.basename(update_location)),
update_location + '/', self._user_location + '/'])
[gentoo-commits] proj/portage:master commit in: lib/portage/repository/storage/, lib/portage/sync/, ...
commit: 884ad951d700d1871cab2e321e4d8635b1a0f698
Author: Zac Medico gentoo org>
AuthorDate: Mon Jul 30 06:21:30 2018 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Sep 24 04:24:10 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=884ad951
rsync: split out repo storage framework
Since there are many ways to manage repository storage, split out a repo
storage framework. The HardlinkQuarantineRepoStorage class implements
the existing default behavior, and the InplaceRepoStorage class
implements the legacy behavior (when sync-allow-hardlinks is disabled in
repos.conf).
Each class implements RepoStorageInterface, which uses coroutine methods
since coroutines are well-suited to the I/O bound tasks that these
methods perform. The _sync_decorator is used to convert coroutine
methods to synchronous methods, for smooth integration into the
surrounding synchronous code.
Bug: https://bugs.gentoo.org/662070
Reviewed-by: Brian Dolbec gentoo.org>
Signed-off-by: Zac Medico gentoo.org>
lib/portage/repository/storage/__init__.py | 2 +
.../repository/storage/hardlink_quarantine.py | 95 ++
lib/portage/repository/storage/inplace.py | 49 +++
lib/portage/repository/storage/interface.py| 87
lib/portage/sync/controller.py | 1 +
lib/portage/sync/modules/rsync/rsync.py| 85 +--
lib/portage/sync/syncbase.py | 53 +++-
7 files changed, 306 insertions(+), 66 deletions(-)
diff --git a/lib/portage/repository/storage/__init__.py
b/lib/portage/repository/storage/__init__.py
new file mode 100644
index 0..58496758f
--- /dev/null
+++ b/lib/portage/repository/storage/__init__.py
@@ -0,0 +1,2 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
diff --git a/lib/portage/repository/storage/hardlink_quarantine.py
b/lib/portage/repository/storage/hardlink_quarantine.py
new file mode 100644
index 0..7e9cf4493
--- /dev/null
+++ b/lib/portage/repository/storage/hardlink_quarantine.py
@@ -0,0 +1,95 @@
+# Copyright 2018 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage import os
+from portage.repository.storage.interface import (
+ RepoStorageException,
+ RepoStorageInterface,
+)
+from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import (
+ coroutine,
+ coroutine_return,
+)
+
+from _emerge.SpawnProcess import SpawnProcess
+
+
+class HardlinkQuarantineRepoStorage(RepoStorageInterface):
+ """
+ This is the default storage module, since its quite compatible with
+ most configurations.
+
+ It's desirable to be able to create shared hardlinks between the
+ download directory and the normal repository, and this is facilitated
+ by making the download directory be a subdirectory of the normal
+ repository location (ensuring that no mountpoints are crossed).
+ Shared hardlinks are created by using the rsync --link-dest option.
+
+ Since the download is initially unverified, it is safest to save
+ it in a quarantine directory. The quarantine directory is also
+ useful for making the repository update more atomic, so that it
+ less likely that normal repository location will be observed in
+ a partially synced state.
+ """
+ def __init__(self, repo, spawn_kwargs):
+ self._user_location = repo.location
+ self._update_location = None
+ self._spawn_kwargs = spawn_kwargs
+ self._current_update = None
+
+ @coroutine
+ def _check_call(self, cmd):
+ """
+ Run cmd and raise RepoStorageException on failure.
+
+ @param cmd: command to executre
+ @type cmd: list
+ """
+ p = SpawnProcess(args=cmd, scheduler=asyncio._wrap_loop(),
**self._spawn_kwargs)
+ p.start()
+ if (yield p.async_wait()) != os.EX_OK:
+ raise RepoStorageException('command exited with status
{}: {}'.\
+ format(p.returncode, ' '.join(cmd)))
+
+ @coroutine
+ def init_update(self):
+ update_location = os.path.join(self._user_location,
'.tmp-unverified-download-quarantine')
+ yield self._check_call(['rm', '-rf', update_location])
+
+ # Use rsync --link-dest to hardlink a files into
self._update_location,
+ # since cp -l is not portable.
+ yield self._check_call(['rsync', '-a', '--link-dest',
self._user_location,
+ '--exclude',
'/{}'.format(os.path.basename(update_location)),
+ self._user_location + '/', update_location + '/'])
+
+ self._update_location =
