Most of you know about the plugin-sync branch of our portage.git repo.
I believe it is ready for review and merge into master to become the
new sync code for the future.
Rather than follow this with 49 patch mails ... I produced a final git
diff of all the changes. I have cleaned up and squashed the original
commits to a point I do not want to squash further. This will keep
enough development history for important changes for future
troubleshooting if it is needed. Instead of inlining this diff like
most people prefer, I have attached it. It is 4.4K+ lines long.
Probably too long for many email clients. Plus opeing it in an editor
will allow the editors syntax highlighting to make code review easier.
I have fully rebased the code on the current master (commit 582cb806f88
QA patch) and pushed it to both our gentoo repo and our github
gentoo/portage repo. For those that wish to review it using githubs
interface. You have that option too.
Thank you to all those that contributed to the code and testing.
--
Brian Dolbec <dolsen>
diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync
index 2f0689c..43f92b6 100755
--- a/bin/emerge-webrsync
+++ b/bin/emerge-webrsync
@@ -501,8 +501,8 @@ main() {
# This is a sanity check to help prevent people like funtoo users
# from accidentally wiping out their git tree.
- if [[ -n ${repo_sync_type} && ${repo_sync_type} != rsync ]] ; then
- echo "The current sync-type attribute of repository 'gentoo' is not set to 'rsync':" >&2
+ if [ [ -n ${repo_sync_type} ] && [ ${repo_sync_type} != rsync -o ${repo_sync_type} != websync ] ] ; then
+ echo "The current sync-type attribute of repository 'gentoo' is not set to 'rsync' or 'websync':" >&2
echo >&2
echo " sync-type=${repo_sync_type}" >&2
echo >&2
diff --git a/cnf/repos.conf b/cnf/repos.conf
index 8c657da..1ca98ca 100644
--- a/cnf/repos.conf
+++ b/cnf/repos.conf
@@ -5,3 +5,4 @@ main-repo = gentoo
location = /usr/portage
sync-type = rsync
sync-uri = rsync://rsync.gentoo.org/gentoo-portage
+auto-sync = yes
diff --git a/man/emaint.1 b/man/emaint.1
index 8356299..f02bc68 100644
--- a/man/emaint.1
+++ b/man/emaint.1
@@ -37,6 +37,9 @@ Perform package move updates for binary packages located in \fBPKGDIR\fR.
.BR moveinst
Perform package move updates for installed packages.
.TP
+.BR sync
+Perform sync actions on specified repositories.
+.TP
.BR world
Fix problems in the \fIworld\fR file.
.SH DEFAULT OPTIONS
@@ -46,7 +49,7 @@ Check for any problems that may exist. (all commands)
.TP
.B \-f, \-\-fix
Fix any problems that may exist. (not all commands)
-.SH OPTIONS
+.SH OPTIONS logs command
.TP
.B \-C, \-\-clean
Cleans the logs from \fBPORT_LOGDIR\fR (logs command only)
@@ -58,6 +61,16 @@ OPTION (logs command only)
.B \-t NUM, \-\-time NUM
Changes the minimum age \fBNUM\fR (in days) of the logs to be listed or
deleted. (logs command only)
+.SH OPTIONS sync command
+.TP
+.B \-a, \-\-auto
+Sync repositories which have its auto\-sync setting set yes, true. (sync command only)
+.TP
+.B \-A, \-\-allrepos
+Sync all repositories which have a sync\-uri specified. (sync command only)
+.TP
+.B \-r, \-\-repo REPO
+Sync the repository specified. (sync command only)
.SH "REPORTING BUGS"
Please report bugs via http://bugs.gentoo.org/
.SH AUTHORS
diff --git a/man/emerge.1 b/man/emerge.1
index e32bb43..f197984 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -240,7 +240,7 @@ the package name. \fBTake caution\fR as the descriptions are also
matched as regular expressions.
.TP
.BR \-\-sync
-Updates repositories, for which sync\-type and sync\-uri attributes are
+Updates repositories, for which auto\-sync, sync\-type and sync\-uri attributes are
set in repos.conf. See \fBportage\fR(5) for more information.
The \fBPORTAGE_SYNC_STALE\fR variable configures
warnings that are shown when emerge \-\-sync has not
@@ -251,6 +251,12 @@ The emerge \-\-sync action will revert local changes (e.g. modifications or
additions of files) inside repositories synchronized using rsync.
\fBNOTE:\fR
+The emerge \-\-sync command is a compatibility command. Sync operations are
+now performed using the the new emaint sync module. This new emaint sync module
+has greater functionality and flexibility. Please refer to \fBemaint(1)\fR for
+more information about sync operations.
+
+\fBNOTE:\fR
The \fBemerge\-webrsync\fR program will download the entire
portage tree as a tarball, which is much faster than emerge
\-\-sync for first time syncs.
diff --git a/man/portage.5 b/man/portage.5
index e399f0f..efafc27 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -818,6 +818,16 @@ When 'force = aliases' attribute is not set, \fBegencache\fR(1),
since operations performed by these tools are inherently
\fBnot\fR \fIsite\-specific\fR.
.TP
+.B auto\-sync
+This setting determines if the repo will be synced during "\fBemerge \-\-sync\fR" or
+"\fBemaint sync \-\-auto\fR" runs. This allows for repositories to be synced only when
+desired via "\fBemaint sync \-\-repo foo\fR".
+.br
+Valid values: yes, no, true, false.
+.br
+If unset, the repo will be treated as set
+no, false.
+.TP
.B eclass\-overrides
Makes given repository inherit eclasses from specified repositories.
.br
diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index fa04e10..bd068a7 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -27,13 +27,13 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.debug',
'portage.news:count_unread_news,display_news_notifications',
'portage.util._get_vm_info:get_vm_info',
+ 'portage.emaint.modules.sync.sync:SyncRepos',
'_emerge.chk_updated_cfg_files:chk_updated_cfg_files',
'_emerge.help:help@emerge_help',
'_emerge.post_emerge:display_news_notification,post_emerge',
'_emerge.stdout_spinner:stdout_spinner',
)
-from portage.localization import _
from portage import os
from portage import shutil
from portage import eapi_is_supported, _encodings, _unicode_decode
@@ -45,7 +45,7 @@ from portage.dbapi._expand_new_virt import expand_new_virt
from portage.dep import Atom
from portage.eclass_cache import hashed_path
from portage.exception import InvalidAtom, InvalidData, ParseError
-from portage.output import blue, bold, colorize, create_color_func, darkgreen, \
+from portage.output import blue, colorize, create_color_func, darkgreen, \
red, xtermTitle, xtermTitleReset, yellow
good = create_color_func("GOOD")
bad = create_color_func("BAD")
@@ -62,6 +62,8 @@ from portage.util._async.run_main_scheduler import run_main_scheduler
from portage.util._async.SchedulerInterface import SchedulerInterface
from portage.util._eventloop.global_event_loop import global_event_loop
from portage._global_updates import _global_updates
+from portage.sync.old_tree_timestamp import old_tree_timestamp_warn
+from portage.metadata import action_metadata
from _emerge.clear_caches import clear_caches
from _emerge.countdown import countdown
@@ -79,8 +81,6 @@ from _emerge.Scheduler import Scheduler
from _emerge.search import search
from _emerge.SetArg import SetArg
from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice
-from _emerge.sync.getaddrinfo_validate import getaddrinfo_validate
-from _emerge.sync.old_tree_timestamp import old_tree_timestamp_warn
from _emerge.unmerge import unmerge
from _emerge.UnmergeDepPriority import UnmergeDepPriority
from _emerge.UseFlagDisplay import pkg_use_display
@@ -1798,200 +1798,6 @@ def action_info(settings, trees, myopts, myfiles):
tree="bintree")
shutil.rmtree(tmpdir)
-def action_metadata(settings, portdb, myopts, porttrees=None):
- if porttrees is None:
- porttrees = portdb.porttrees
- portage.writemsg_stdout("\n>>> Updating Portage cache\n")
- old_umask = os.umask(0o002)
- cachedir = os.path.normpath(settings.depcachedir)
- if cachedir in ["/", "/bin", "/dev", "/etc", "/home",
- "/lib", "/opt", "/proc", "/root", "/sbin",
- "/sys", "/tmp", "/usr", "/var"]:
- print("!!! PORTAGE_DEPCACHEDIR IS SET TO A PRIMARY " + \
- "ROOT DIRECTORY ON YOUR SYSTEM.", file=sys.stderr)
- print("!!! This is ALMOST CERTAINLY NOT what you want: '%s'" % cachedir, file=sys.stderr)
- sys.exit(73)
- if not os.path.exists(cachedir):
- os.makedirs(cachedir)
-
- auxdbkeys = portdb._known_keys
-
- class TreeData(object):
- __slots__ = ('dest_db', 'eclass_db', 'path', 'src_db', 'valid_nodes')
- def __init__(self, dest_db, eclass_db, path, src_db):
- self.dest_db = dest_db
- self.eclass_db = eclass_db
- self.path = path
- self.src_db = src_db
- self.valid_nodes = set()
-
- porttrees_data = []
- for path in porttrees:
- src_db = portdb._pregen_auxdb.get(path)
- if src_db is None:
- # portdbapi does not populate _pregen_auxdb
- # when FEATURES=metadata-transfer is enabled
- src_db = portdb._create_pregen_cache(path)
-
- if src_db is not None:
- porttrees_data.append(TreeData(portdb.auxdb[path],
- portdb.repositories.get_repo_for_location(path).eclass_db, path, src_db))
-
- porttrees = [tree_data.path for tree_data in porttrees_data]
-
- quiet = settings.get('TERM') == 'dumb' or \
- '--quiet' in myopts or \
- not sys.stdout.isatty()
-
- onProgress = None
- if not quiet:
- progressBar = portage.output.TermProgressBar()
- progressHandler = ProgressHandler()
- onProgress = progressHandler.onProgress
- def display():
- progressBar.set(progressHandler.curval, progressHandler.maxval)
- progressHandler.display = display
- def sigwinch_handler(signum, frame):
- lines, progressBar.term_columns = \
- portage.output.get_term_size()
- signal.signal(signal.SIGWINCH, sigwinch_handler)
-
- # Temporarily override portdb.porttrees so portdb.cp_all()
- # will only return the relevant subset.
- portdb_porttrees = portdb.porttrees
- portdb.porttrees = porttrees
- try:
- cp_all = portdb.cp_all()
- finally:
- portdb.porttrees = portdb_porttrees
-
- curval = 0
- maxval = len(cp_all)
- if onProgress is not None:
- onProgress(maxval, curval)
-
- # TODO: Display error messages, but do not interfere with the progress bar.
- # Here's how:
- # 1) erase the progress bar
- # 2) show the error message
- # 3) redraw the progress bar on a new line
-
- for cp in cp_all:
- for tree_data in porttrees_data:
-
- src_chf = tree_data.src_db.validation_chf
- dest_chf = tree_data.dest_db.validation_chf
- dest_chf_key = '_%s_' % dest_chf
- dest_chf_getter = operator.attrgetter(dest_chf)
-
- for cpv in portdb.cp_list(cp, mytree=tree_data.path):
- tree_data.valid_nodes.add(cpv)
- try:
- src = tree_data.src_db[cpv]
- except (CacheError, KeyError):
- continue
-
- ebuild_location = portdb.findname(cpv, mytree=tree_data.path)
- if ebuild_location is None:
- continue
- ebuild_hash = hashed_path(ebuild_location)
-
- try:
- if not tree_data.src_db.validate_entry(src,
- ebuild_hash, tree_data.eclass_db):
- continue
- except CacheError:
- continue
-
- eapi = src.get('EAPI')
- if not eapi:
- eapi = '0'
- eapi_supported = eapi_is_supported(eapi)
- if not eapi_supported:
- continue
-
- dest = None
- try:
- dest = tree_data.dest_db[cpv]
- except (KeyError, CacheError):
- pass
-
- for d in (src, dest):
- if d is not None and d.get('EAPI') in ('', '0'):
- del d['EAPI']
-
- if src_chf != 'mtime':
- # src may contain an irrelevant _mtime_ which corresponds
- # to the time that the cache entry was written
- src.pop('_mtime_', None)
-
- if src_chf != dest_chf:
- # populate src entry with dest_chf_key
- # (the validity of the dest_chf that we generate from the
- # ebuild here relies on the fact that we already used
- # validate_entry to validate the ebuild with src_chf)
- src[dest_chf_key] = dest_chf_getter(ebuild_hash)
-
- if dest is not None:
- if not (dest[dest_chf_key] == src[dest_chf_key] and \
- tree_data.eclass_db.validate_and_rewrite_cache(
- dest['_eclasses_'], tree_data.dest_db.validation_chf,
- tree_data.dest_db.store_eclass_paths) is not None and \
- set(dest['_eclasses_']) == set(src['_eclasses_'])):
- dest = None
- else:
- # We don't want to skip the write unless we're really
- # sure that the existing cache is identical, so don't
- # trust _mtime_ and _eclasses_ alone.
- for k in auxdbkeys:
- if dest.get(k, '') != src.get(k, ''):
- dest = None
- break
-
- if dest is not None:
- # The existing data is valid and identical,
- # so there's no need to overwrite it.
- continue
-
- try:
- tree_data.dest_db[cpv] = src
- except CacheError:
- # ignore it; can't do anything about it.
- pass
-
- curval += 1
- if onProgress is not None:
- onProgress(maxval, curval)
-
- if onProgress is not None:
- onProgress(maxval, curval)
-
- for tree_data in porttrees_data:
- try:
- dead_nodes = set(tree_data.dest_db)
- except CacheError as e:
- writemsg_level("Error listing cache entries for " + \
- "'%s': %s, continuing...\n" % (tree_data.path, e),
- level=logging.ERROR, noiselevel=-1)
- del e
- else:
- dead_nodes.difference_update(tree_data.valid_nodes)
- for cpv in dead_nodes:
- try:
- del tree_data.dest_db[cpv]
- except (KeyError, CacheError):
- pass
-
- if not quiet:
- # make sure the final progress is displayed
- progressHandler.display()
- print()
- signal.signal(signal.SIGWINCH, signal.SIG_DFL)
-
- portdb.flush_cache()
- sys.stdout.flush()
- os.umask(old_umask)
-
def action_regen(settings, portdb, max_jobs, max_load):
xterm_titles = "notitles" not in settings.features
emergelog(xterm_titles, " === regen")
@@ -2035,668 +1841,15 @@ def action_sync(emerge_config, trees=DeprecationWarning,
emerge_config = load_emerge_config(
action=action, args=[], trees=trees, opts=opts)
- xterm_titles = "notitles" not in \
- emerge_config.target_config.settings.features
- emergelog(xterm_titles, " === sync")
-
- selected_repos = []
- unknown_repo_names = []
- missing_sync_type = []
- if emerge_config.args:
- for repo_name in emerge_config.args:
- try:
- repo = emerge_config.target_config.settings.repositories[repo_name]
- except KeyError:
- unknown_repo_names.append(repo_name)
- else:
- selected_repos.append(repo)
- if repo.sync_type is None:
- missing_sync_type.append(repo)
+ syncer = SyncRepos(emerge_config)
- if unknown_repo_names:
- writemsg_level("!!! %s\n" % _("Unknown repo(s): %s") %
- " ".join(unknown_repo_names),
- level=logging.ERROR, noiselevel=-1)
-
- if missing_sync_type:
- writemsg_level("!!! %s\n" %
- _("Missing sync-type for repo(s): %s") %
- " ".join(repo.name for repo in missing_sync_type),
- level=logging.ERROR, noiselevel=-1)
- if unknown_repo_names or missing_sync_type:
- return 1
-
- else:
- selected_repos.extend(emerge_config.target_config.settings.repositories)
-
- for repo in selected_repos:
- if repo.sync_type is not None:
- returncode = _sync_repo(emerge_config, repo)
- if returncode != os.EX_OK:
- return returncode
-
- # Reload the whole config from scratch.
- portage._sync_mode = False
- load_emerge_config(emerge_config=emerge_config)
- adjust_configs(emerge_config.opts, emerge_config.trees)
+ retvals = syncer.auto_sync(options={'return-messages': False})
- if emerge_config.opts.get('--package-moves') != 'n' and \
- _global_updates(emerge_config.trees,
- emerge_config.target_config.mtimedb["updates"],
- quiet=("--quiet" in emerge_config.opts)):
- emerge_config.target_config.mtimedb.commit()
- # Reload the whole config from scratch.
- load_emerge_config(emerge_config=emerge_config)
- adjust_configs(emerge_config.opts, emerge_config.trees)
-
- mybestpv = emerge_config.target_config.trees['porttree'].dbapi.xmatch(
- "bestmatch-visible", portage.const.PORTAGE_PACKAGE_ATOM)
- mypvs = portage.best(
- emerge_config.target_config.trees['vartree'].dbapi.match(
- portage.const.PORTAGE_PACKAGE_ATOM))
-
- chk_updated_cfg_files(emerge_config.target_config.root,
- portage.util.shlex_split(
- emerge_config.target_config.settings.get("CONFIG_PROTECT", "")))
-
- if mybestpv != mypvs and "--quiet" not in emerge_config.opts:
- print()
- print(warn(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended")
- print(warn(" * ")+"that you update portage now, before any other packages are updated.")
- print()
- print(warn(" * ")+"To update portage, run 'emerge --oneshot portage' now.")
- print()
-
- display_news_notification(emerge_config.target_config, emerge_config.opts)
+ if retvals:
+ return retvals[0][1]
return os.EX_OK
-def _sync_repo(emerge_config, repo):
- settings, trees, mtimedb = emerge_config
- myopts = emerge_config.opts
- enter_invalid = '--ask-enter-invalid' in myopts
- xterm_titles = "notitles" not in settings.features
- msg = ">>> Synchronization of repository '%s' located in '%s'..." % (repo.name, repo.location)
- emergelog(xterm_titles, msg)
- writemsg_level(msg + "\n")
- out = portage.output.EOutput()
- try:
- st = os.stat(repo.location)
- except OSError:
- st = None
- if st is None:
- print(">>> '%s' not found, creating it." % repo.location)
- portage.util.ensure_dirs(repo.location, mode=0o755)
- st = os.stat(repo.location)
-
- usersync_uid = None
- spawn_kwargs = {}
- spawn_kwargs["env"] = settings.environ()
- if 'usersync' in settings.features and \
- portage.data.secpass >= 2 and \
- (st.st_uid != os.getuid() and st.st_mode & 0o700 or \
- st.st_gid != os.getgid() and st.st_mode & 0o070):
- try:
- homedir = pwd.getpwuid(st.st_uid).pw_dir
- except KeyError:
- pass
- else:
- # Drop privileges when syncing, in order to match
- # existing uid/gid settings.
- usersync_uid = st.st_uid
- spawn_kwargs["uid"] = st.st_uid
- spawn_kwargs["gid"] = st.st_gid
- spawn_kwargs["groups"] = [st.st_gid]
- spawn_kwargs["env"]["HOME"] = homedir
- umask = 0o002
- if not st.st_mode & 0o020:
- umask = umask | 0o020
- spawn_kwargs["umask"] = umask
-
- if usersync_uid is not None:
- # PORTAGE_TMPDIR is used below, so validate it and
- # bail out if necessary.
- rval = _check_temp_dir(settings)
- if rval != os.EX_OK:
- return rval
-
- syncuri = repo.sync_uri
-
- vcs_dirs = frozenset(VCS_DIRS)
- vcs_dirs = vcs_dirs.intersection(os.listdir(repo.location))
-
- os.umask(0o022)
- dosyncuri = syncuri
- updatecache_flg = False
- if repo.sync_type == "git":
- # Update existing git repository, and ignore the syncuri. We are
- # going to trust the user and assume that the user is in the branch
- # that he/she wants updated. We'll let the user manage branches with
- # git directly.
- if portage.process.find_binary("git") is None:
- msg = ["Command not found: git",
- "Type \"emerge %s\" to enable git support." % portage.const.GIT_PACKAGE_ATOM]
- for l in msg:
- writemsg_level("!!! %s\n" % l,
- level=logging.ERROR, noiselevel=-1)
- return 1
- msg = ">>> Starting git pull in %s..." % repo.location
- emergelog(xterm_titles, msg )
- writemsg_level(msg + "\n")
- exitcode = portage.process.spawn_bash("cd %s ; git pull" % \
- (portage._shell_quote(repo.location),),
- **portage._native_kwargs(spawn_kwargs))
- if exitcode != os.EX_OK:
- msg = "!!! git pull error in %s." % repo.location
- emergelog(xterm_titles, msg)
- writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
- return exitcode
- msg = ">>> Git pull in %s successful" % repo.location
- emergelog(xterm_titles, msg)
- writemsg_level(msg + "\n")
- elif repo.sync_type == "rsync":
- for vcs_dir in vcs_dirs:
- writemsg_level(("!!! %s appears to be under revision " + \
- "control (contains %s).\n!!! Aborting rsync sync.\n") % \
- (repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
- return 1
- rsync_binary = portage.process.find_binary("rsync")
- if rsync_binary is None:
- print("!!! /usr/bin/rsync does not exist, so rsync support is disabled.")
- print("!!! Type \"emerge %s\" to enable rsync support." % portage.const.RSYNC_PACKAGE_ATOM)
- return os.EX_UNAVAILABLE
- mytimeout=180
-
- rsync_opts = []
- if settings["PORTAGE_RSYNC_OPTS"] == "":
- portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
- rsync_opts.extend([
- "--recursive", # Recurse directories
- "--links", # Consider symlinks
- "--safe-links", # Ignore links outside of tree
- "--perms", # Preserve permissions
- "--times", # Preserive mod times
- "--omit-dir-times",
- "--compress", # Compress the data transmitted
- "--force", # Force deletion on non-empty dirs
- "--whole-file", # Don't do block transfers, only entire files
- "--delete", # Delete files that aren't in the master tree
- "--stats", # Show final statistics about what was transfered
- "--human-readable",
- "--timeout="+str(mytimeout), # IO timeout if not done in X seconds
- "--exclude=/distfiles", # Exclude distfiles from consideration
- "--exclude=/local", # Exclude local from consideration
- "--exclude=/packages", # Exclude packages from consideration
- ])
-
- else:
- # The below validation is not needed when using the above hardcoded
- # defaults.
-
- portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
- rsync_opts.extend(portage.util.shlex_split(
- settings.get("PORTAGE_RSYNC_OPTS", "")))
- for opt in ("--recursive", "--times"):
- if opt not in rsync_opts:
- portage.writemsg(yellow("WARNING:") + " adding required option " + \
- "%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
- rsync_opts.append(opt)
-
- for exclude in ("distfiles", "local", "packages"):
- opt = "--exclude=/%s" % exclude
- if opt not in rsync_opts:
- portage.writemsg(yellow("WARNING:") + \
- " adding required option %s not included in " % opt + \
- "PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
- rsync_opts.append(opt)
-
- if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
- def rsync_opt_startswith(opt_prefix):
- for x in rsync_opts:
- if x.startswith(opt_prefix):
- return True
- return False
-
- if not rsync_opt_startswith("--timeout="):
- rsync_opts.append("--timeout=%d" % mytimeout)
-
- for opt in ("--compress", "--whole-file"):
- if opt not in rsync_opts:
- portage.writemsg(yellow("WARNING:") + " adding required option " + \
- "%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
- rsync_opts.append(opt)
-
- if "--quiet" in myopts:
- rsync_opts.append("--quiet") # Shut up a lot
- else:
- rsync_opts.append("--verbose") # Print filelist
-
- if "--verbose" in myopts:
- rsync_opts.append("--progress") # Progress meter for each file
-
- if "--debug" in myopts:
- rsync_opts.append("--checksum") # Force checksum on all files
-
- # Real local timestamp file.
- servertimestampfile = os.path.join(
- repo.location, "metadata", "timestamp.chk")
-
- content = portage.util.grabfile(servertimestampfile)
- mytimestamp = 0
- if content:
- try:
- mytimestamp = time.mktime(time.strptime(content[0],
- TIMESTAMP_FORMAT))
- except (OverflowError, ValueError):
- pass
- del content
-
- try:
- rsync_initial_timeout = \
- int(settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
- except ValueError:
- rsync_initial_timeout = 15
-
- try:
- maxretries=int(settings["PORTAGE_RSYNC_RETRIES"])
- except SystemExit as e:
- raise # Needed else can't exit
- except:
- maxretries = -1 #default number of retries
-
- retries=0
- try:
- proto, user_name, hostname, port = re.split(
- r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
- syncuri, maxsplit=4)[1:5]
- except ValueError:
- writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
- noiselevel=-1, level=logging.ERROR)
- return 1
-
- ssh_opts = settings.get("PORTAGE_SSH_OPTS")
-
- if port is None:
- port=""
- if user_name is None:
- user_name=""
- if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
- getaddrinfo_host = hostname
- else:
- # getaddrinfo needs the brackets stripped
- getaddrinfo_host = hostname[1:-1]
- updatecache_flg=True
- all_rsync_opts = set(rsync_opts)
- extra_rsync_opts = portage.util.shlex_split(
- settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
- all_rsync_opts.update(extra_rsync_opts)
-
- family = socket.AF_UNSPEC
- if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
- family = socket.AF_INET
- elif socket.has_ipv6 and \
- ("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
- family = socket.AF_INET6
-
- addrinfos = None
- uris = []
-
- try:
- addrinfos = getaddrinfo_validate(
- socket.getaddrinfo(getaddrinfo_host, None,
- family, socket.SOCK_STREAM))
- except socket.error as e:
- writemsg_level(
- "!!! getaddrinfo failed for '%s': %s\n" % (hostname,
- _unicode_decode(e.strerror, encoding=_encodings['stdio'])),
- noiselevel=-1, level=logging.ERROR)
-
- if addrinfos:
-
- AF_INET = socket.AF_INET
- AF_INET6 = None
- if socket.has_ipv6:
- AF_INET6 = socket.AF_INET6
-
- ips_v4 = []
- ips_v6 = []
-
- for addrinfo in addrinfos:
- if addrinfo[0] == AF_INET:
- ips_v4.append("%s" % addrinfo[4][0])
- elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
- # IPv6 addresses need to be enclosed in square brackets
- ips_v6.append("[%s]" % addrinfo[4][0])
-
- random.shuffle(ips_v4)
- random.shuffle(ips_v6)
-
- # Give priority to the address family that
- # getaddrinfo() returned first.
- if AF_INET6 is not None and addrinfos and \
- addrinfos[0][0] == AF_INET6:
- ips = ips_v6 + ips_v4
- else:
- ips = ips_v4 + ips_v6
-
- for ip in ips:
- uris.append(syncuri.replace(
- "//" + user_name + hostname + port + "/",
- "//" + user_name + ip + port + "/", 1))
-
- if not uris:
- # With some configurations we need to use the plain hostname
- # rather than try to resolve the ip addresses (bug #340817).
- uris.append(syncuri)
-
- # reverse, for use with pop()
- uris.reverse()
-
- effective_maxretries = maxretries
- if effective_maxretries < 0:
- effective_maxretries = len(uris) - 1
-
- SERVER_OUT_OF_DATE = -1
- EXCEEDED_MAX_RETRIES = -2
- while (1):
- if uris:
- dosyncuri = uris.pop()
- else:
- writemsg("!!! Exhausted addresses for %s\n" % \
- hostname, noiselevel=-1)
- return 1
-
- if (retries==0):
- if "--ask" in myopts:
- uq = UserQuery(myopts)
- if uq.query("Do you want to sync your Portage tree " + \
- "with the mirror at\n" + blue(dosyncuri) + bold("?"),
- enter_invalid) == "No":
- print()
- print("Quitting.")
- print()
- sys.exit(128 + signal.SIGINT)
- emergelog(xterm_titles, ">>> Starting rsync with " + dosyncuri)
- if "--quiet" not in myopts:
- print(">>> Starting rsync with "+dosyncuri+"...")
- else:
- emergelog(xterm_titles,
- ">>> Starting retry %d of %d with %s" % \
- (retries, effective_maxretries, dosyncuri))
- writemsg_stdout(
- "\n\n>>> Starting retry %d of %d with %s\n" % \
- (retries, effective_maxretries, dosyncuri), noiselevel=-1)
-
- if dosyncuri.startswith('ssh://'):
- dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
-
- if mytimestamp != 0 and "--quiet" not in myopts:
- print(">>> Checking server timestamp ...")
-
- rsynccommand = [rsync_binary] + rsync_opts + extra_rsync_opts
-
- if proto == 'ssh' and ssh_opts:
- rsynccommand.append("--rsh=ssh " + ssh_opts)
-
- if "--debug" in myopts:
- print(rsynccommand)
-
- exitcode = os.EX_OK
- servertimestamp = 0
- # Even if there's no timestamp available locally, fetch the
- # timestamp anyway as an initial probe to verify that the server is
- # responsive. This protects us from hanging indefinitely on a
- # connection attempt to an unresponsive server which rsync's
- # --timeout option does not prevent.
- if True:
- # Temporary file for remote server timestamp comparison.
- # NOTE: If FEATURES=usersync is enabled then the tempfile
- # needs to be in a directory that's readable by the usersync
- # user. We assume that PORTAGE_TMPDIR will satisfy this
- # requirement, since that's not necessarily true for the
- # default directory used by the tempfile module.
- if usersync_uid is not None:
- tmpdir = settings['PORTAGE_TMPDIR']
- else:
- # use default dir from tempfile module
- tmpdir = None
- fd, tmpservertimestampfile = \
- tempfile.mkstemp(dir=tmpdir)
- os.close(fd)
- if usersync_uid is not None:
- portage.util.apply_permissions(tmpservertimestampfile,
- uid=usersync_uid)
- mycommand = rsynccommand[:]
- mycommand.append(dosyncuri.rstrip("/") + \
- "/metadata/timestamp.chk")
- mycommand.append(tmpservertimestampfile)
- content = None
- mypids = []
- try:
- # Timeout here in case the server is unresponsive. The
- # --timeout rsync option doesn't apply to the initial
- # connection attempt.
- try:
- if rsync_initial_timeout:
- portage.exception.AlarmSignal.register(
- rsync_initial_timeout)
-
- mypids.extend(portage.process.spawn(
- mycommand, returnpid=True,
- **portage._native_kwargs(spawn_kwargs)))
- exitcode = os.waitpid(mypids[0], 0)[1]
- if usersync_uid is not None:
- portage.util.apply_permissions(tmpservertimestampfile,
- uid=os.getuid())
- content = portage.grabfile(tmpservertimestampfile)
- finally:
- if rsync_initial_timeout:
- portage.exception.AlarmSignal.unregister()
- try:
- os.unlink(tmpservertimestampfile)
- except OSError:
- pass
- except portage.exception.AlarmSignal:
- # timed out
- print('timed out')
- # With waitpid and WNOHANG, only check the
- # first element of the tuple since the second
- # element may vary (bug #337465).
- if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
- os.kill(mypids[0], signal.SIGTERM)
- os.waitpid(mypids[0], 0)
- # This is the same code rsync uses for timeout.
- exitcode = 30
- else:
- if exitcode != os.EX_OK:
- if exitcode & 0xff:
- exitcode = (exitcode & 0xff) << 8
- else:
- exitcode = exitcode >> 8
-
- if content:
- try:
- servertimestamp = time.mktime(time.strptime(
- content[0], TIMESTAMP_FORMAT))
- except (OverflowError, ValueError):
- pass
- del mycommand, mypids, content
- if exitcode == os.EX_OK:
- if (servertimestamp != 0) and (servertimestamp == mytimestamp):
- emergelog(xterm_titles,
- ">>> Cancelling sync -- Already current.")
- print()
- print(">>>")
- print(">>> Timestamps on the server and in the local repository are the same.")
- print(">>> Cancelling all further sync action. You are already up to date.")
- print(">>>")
- print(">>> In order to force sync, remove '%s'." % servertimestampfile)
- print(">>>")
- print()
- return os.EX_OK
- elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
- emergelog(xterm_titles,
- ">>> Server out of date: %s" % dosyncuri)
- print()
- print(">>>")
- print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
- print(">>>")
- print(">>> In order to force sync, remove '%s'." % servertimestampfile)
- print(">>>")
- print()
- exitcode = SERVER_OUT_OF_DATE
- elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
- # actual sync
- mycommand = rsynccommand + [dosyncuri+"/", repo.location]
- exitcode = None
- try:
- exitcode = portage.process.spawn(mycommand,
- **portage._native_kwargs(spawn_kwargs))
- finally:
- if exitcode is None:
- # interrupted
- exitcode = 128 + signal.SIGINT
-
- # 0 Success
- # 1 Syntax or usage error
- # 2 Protocol incompatibility
- # 5 Error starting client-server protocol
- # 35 Timeout waiting for daemon connection
- if exitcode not in (0, 1, 2, 5, 35):
- # If the exit code is not among those listed above,
- # then we may have a partial/inconsistent sync
- # state, so our previously read timestamp as well
- # as the corresponding file can no longer be
- # trusted.
- mytimestamp = 0
- try:
- os.unlink(servertimestampfile)
- except OSError:
- pass
-
- if exitcode in [0,1,3,4,11,14,20,21]:
- break
- elif exitcode in [1,3,4,11,14,20,21]:
- break
- else:
- # Code 2 indicates protocol incompatibility, which is expected
- # for servers with protocol < 29 that don't support
- # --prune-empty-directories. Retry for a server that supports
- # at least rsync protocol version 29 (>=rsync-2.6.4).
- pass
-
- retries=retries+1
-
- if maxretries < 0 or retries <= maxretries:
- print(">>> Retrying...")
- else:
- # over retries
- # exit loop
- updatecache_flg=False
- exitcode = EXCEEDED_MAX_RETRIES
- break
-
- if (exitcode==0):
- emergelog(xterm_titles, "=== Sync completed with %s" % dosyncuri)
- elif exitcode == SERVER_OUT_OF_DATE:
- return 1
- elif exitcode == EXCEEDED_MAX_RETRIES:
- sys.stderr.write(
- ">>> Exceeded PORTAGE_RSYNC_RETRIES: %s\n" % maxretries)
- return 1
- elif (exitcode>0):
- msg = []
- if exitcode==1:
- msg.append("Rsync has reported that there is a syntax error. Please ensure")
- msg.append("that sync-uri attribute for repository '%s' is proper." % repo.name)
- msg.append("sync-uri: '%s'" % repo.sync_uri)
- elif exitcode==11:
- msg.append("Rsync has reported that there is a File IO error. Normally")
- msg.append("this means your disk is full, but can be caused by corruption")
- msg.append("on the filesystem that contains repository '%s'. Please investigate" % repo.name)
- msg.append("and try again after the problem has been fixed.")
- msg.append("Location of repository: '%s'" % repo.location)
- elif exitcode==20:
- msg.append("Rsync was killed before it finished.")
- else:
- msg.append("Rsync has not successfully finished. It is recommended that you keep")
- msg.append("trying or that you use the 'emerge-webrsync' option if you are unable")
- msg.append("to use rsync due to firewall or other restrictions. This should be a")
- msg.append("temporary problem unless complications exist with your network")
- msg.append("(and possibly your system's filesystem) configuration.")
- for line in msg:
- out.eerror(line)
- return exitcode
- elif repo.sync_type == "cvs":
- if not os.path.exists("/usr/bin/cvs"):
- print("!!! /usr/bin/cvs does not exist, so CVS support is disabled.")
- print("!!! Type \"emerge %s\" to enable CVS support." % portage.const.CVS_PACKAGE_ATOM)
- return os.EX_UNAVAILABLE
- cvs_root = syncuri
- if cvs_root.startswith("cvs://"):
- cvs_root = cvs_root[6:]
- if not os.path.exists(os.path.join(repo.location, "CVS")):
- #initial checkout
- print(">>> Starting initial cvs checkout with "+syncuri+"...")
- try:
- os.rmdir(repo.location)
- except OSError as e:
- if e.errno != errno.ENOENT:
- sys.stderr.write(
- "!!! existing '%s' directory; exiting.\n" % repo.location)
- return 1
- del e
- if portage.process.spawn_bash(
- "cd %s; exec cvs -z0 -d %s co -P -d %s %s" %
- (portage._shell_quote(os.path.dirname(repo.location)), portage._shell_quote(cvs_root),
- portage._shell_quote(os.path.basename(repo.location)), portage._shell_quote(repo.sync_cvs_repo)),
- **portage._native_kwargs(spawn_kwargs)) != os.EX_OK:
- print("!!! cvs checkout error; exiting.")
- return 1
- else:
- #cvs update
- print(">>> Starting cvs update with "+syncuri+"...")
- retval = portage.process.spawn_bash(
- "cd %s; exec cvs -z0 -q update -dP" % \
- (portage._shell_quote(repo.location),),
- **portage._native_kwargs(spawn_kwargs))
- if retval != os.EX_OK:
- writemsg_level("!!! cvs update error; exiting.\n",
- noiselevel=-1, level=logging.ERROR)
- return retval
- dosyncuri = syncuri
-
- # Reload the whole config from scratch.
- settings, trees, mtimedb = load_emerge_config(emerge_config=emerge_config)
- adjust_configs(emerge_config.opts, emerge_config.trees)
- portdb = trees[settings['EROOT']]['porttree'].dbapi
-
- if repo.sync_type == "git":
- # NOTE: Do this after reloading the config, in case
- # it did not exist prior to sync, so that the config
- # and portdb properly account for its existence.
- exitcode = git_sync_timestamps(portdb, repo.location)
- if exitcode == os.EX_OK:
- updatecache_flg = True
-
- if updatecache_flg and "metadata-transfer" not in settings.features:
- updatecache_flg = False
-
- if updatecache_flg and \
- os.path.exists(os.path.join(repo.location, 'metadata', 'cache')):
-
- # Only update cache for repo.location since that's
- # the only one that's been synced here.
- action_metadata(settings, portdb, myopts, porttrees=[repo.location])
-
- postsync = os.path.join(settings["PORTAGE_CONFIGROOT"], portage.USER_CONFIG_PATH, "bin", "post_sync")
- if os.access(postsync, os.X_OK):
- retval = portage.process.spawn([postsync, dosyncuri], env=settings.environ())
- if retval != os.EX_OK:
- writemsg_level(" %s spawn failed of %s\n" % (bad("*"), postsync,),
- level=logging.ERROR, noiselevel=-1)
-
- return os.EX_OK
def action_uninstall(settings, trees, ldpath_mtimes,
opts, action, files, spinner):
@@ -3061,145 +2214,6 @@ def getportageversion(portdir, _unused, profile, chost, vardb):
return "Portage %s (%s, %s, %s, %s, %s)" % \
(portage.VERSION, pythonver, profilever, gccver, ",".join(libcver), unameout)
-def git_sync_timestamps(portdb, portdir):
- """
- Since git doesn't preserve timestamps, synchronize timestamps between
- entries and ebuilds/eclasses. Assume the cache has the correct timestamp
- for a given file as long as the file in the working tree is not modified
- (relative to HEAD).
- """
-
- cache_db = portdb._pregen_auxdb.get(portdir)
-
- try:
- if cache_db is None:
- # portdbapi does not populate _pregen_auxdb
- # when FEATURES=metadata-transfer is enabled
- cache_db = portdb._create_pregen_cache(portdir)
- except CacheError as e:
- writemsg_level("!!! Unable to instantiate cache: %s\n" % (e,),
- level=logging.ERROR, noiselevel=-1)
- return 1
-
- if cache_db is None:
- return os.EX_OK
-
- if cache_db.validation_chf != 'mtime':
- # newer formats like md5-dict do not require mtime sync
- return os.EX_OK
-
- writemsg_level(">>> Synchronizing timestamps...\n")
-
- ec_dir = os.path.join(portdir, "eclass")
- try:
- ec_names = set(f[:-7] for f in os.listdir(ec_dir) \
- if f.endswith(".eclass"))
- except OSError as e:
- writemsg_level("!!! Unable to list eclasses: %s\n" % (e,),
- level=logging.ERROR, noiselevel=-1)
- return 1
-
- args = [portage.const.BASH_BINARY, "-c",
- "cd %s && git diff-index --name-only --diff-filter=M HEAD" % \
- portage._shell_quote(portdir)]
- proc = subprocess.Popen(args, stdout=subprocess.PIPE)
- modified_files = set(_unicode_decode(l).rstrip("\n") for l in proc.stdout)
- rval = proc.wait()
- proc.stdout.close()
- if rval != os.EX_OK:
- return rval
-
- modified_eclasses = set(ec for ec in ec_names \
- if os.path.join("eclass", ec + ".eclass") in modified_files)
-
- updated_ec_mtimes = {}
-
- for cpv in cache_db:
- cpv_split = portage.catpkgsplit(cpv)
- if cpv_split is None:
- writemsg_level("!!! Invalid cache entry: %s\n" % (cpv,),
- level=logging.ERROR, noiselevel=-1)
- continue
-
- cat, pn, ver, rev = cpv_split
- cat, pf = portage.catsplit(cpv)
- relative_eb_path = os.path.join(cat, pn, pf + ".ebuild")
- if relative_eb_path in modified_files:
- continue
-
- try:
- cache_entry = cache_db[cpv]
- eb_mtime = cache_entry.get("_mtime_")
- ec_mtimes = cache_entry.get("_eclasses_")
- except KeyError:
- writemsg_level("!!! Missing cache entry: %s\n" % (cpv,),
- level=logging.ERROR, noiselevel=-1)
- continue
- except CacheError as e:
- writemsg_level("!!! Unable to access cache entry: %s %s\n" % \
- (cpv, e), level=logging.ERROR, noiselevel=-1)
- continue
-
- if eb_mtime is None:
- writemsg_level("!!! Missing ebuild mtime: %s\n" % (cpv,),
- level=logging.ERROR, noiselevel=-1)
- continue
-
- try:
- eb_mtime = long(eb_mtime)
- except ValueError:
- writemsg_level("!!! Invalid ebuild mtime: %s %s\n" % \
- (cpv, eb_mtime), level=logging.ERROR, noiselevel=-1)
- continue
-
- if ec_mtimes is None:
- writemsg_level("!!! Missing eclass mtimes: %s\n" % (cpv,),
- level=logging.ERROR, noiselevel=-1)
- continue
-
- if modified_eclasses.intersection(ec_mtimes):
- continue
-
- missing_eclasses = set(ec_mtimes).difference(ec_names)
- if missing_eclasses:
- writemsg_level("!!! Non-existent eclass(es): %s %s\n" % \
- (cpv, sorted(missing_eclasses)), level=logging.ERROR,
- noiselevel=-1)
- continue
-
- eb_path = os.path.join(portdir, relative_eb_path)
- try:
- current_eb_mtime = os.stat(eb_path)
- except OSError:
- writemsg_level("!!! Missing ebuild: %s\n" % \
- (cpv,), level=logging.ERROR, noiselevel=-1)
- continue
-
- inconsistent = False
- for ec, (ec_path, ec_mtime) in ec_mtimes.items():
- updated_mtime = updated_ec_mtimes.get(ec)
- if updated_mtime is not None and updated_mtime != ec_mtime:
- writemsg_level("!!! Inconsistent eclass mtime: %s %s\n" % \
- (cpv, ec), level=logging.ERROR, noiselevel=-1)
- inconsistent = True
- break
-
- if inconsistent:
- continue
-
- if current_eb_mtime != eb_mtime:
- os.utime(eb_path, (eb_mtime, eb_mtime))
-
- for ec, (ec_path, ec_mtime) in ec_mtimes.items():
- if ec in updated_ec_mtimes:
- continue
- ec_path = os.path.join(ec_dir, ec + ".eclass")
- current_mtime = os.stat(ec_path)[stat.ST_MTIME]
- if current_mtime != ec_mtime:
- os.utime(ec_path, (ec_mtime, ec_mtime))
- updated_ec_mtimes[ec] = ec_mtime
-
- return os.EX_OK
class _emerge_config(SlotObject):
diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py
index 52aa9c5..7e8a800 100644
--- a/pym/_emerge/main.py
+++ b/pym/_emerge/main.py
@@ -1033,11 +1033,13 @@ def emerge_main(args=None):
elif myaction == "moo":
print(COWSAY_MOO % platform.system())
return os.EX_OK
+ elif myaction == "sync":
+ # need to set this to True now in order for the repository config
+ # loading to allow new repos with non-existent directories
+ portage._sync_mode = True
# Portage needs to ensure a sane umask for the files it creates.
os.umask(0o22)
- if myaction == "sync":
- portage._sync_mode = True
emerge_config = load_emerge_config(
action=myaction, args=myfiles, opts=myopts)
rval = profile_check(emerge_config.trees, emerge_config.action)
diff --git a/pym/_emerge/sync/__init__.py b/pym/_emerge/sync/__init__.py
deleted file mode 100644
index 21a391a..0000000
--- a/pym/_emerge/sync/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-# Copyright 2010 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
diff --git a/pym/_emerge/sync/getaddrinfo_validate.py b/pym/_emerge/sync/getaddrinfo_validate.py
deleted file mode 100644
index 5e6009c..0000000
--- a/pym/_emerge/sync/getaddrinfo_validate.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright 2010 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-import sys
-
-if sys.hexversion >= 0x3000000:
- basestring = str
-
-def getaddrinfo_validate(addrinfos):
- """
- Validate structures returned from getaddrinfo(),
- since they may be corrupt, especially when python
- has IPv6 support disabled (bug #340899).
- """
- valid_addrinfos = []
- for addrinfo in addrinfos:
- try:
- if len(addrinfo) != 5:
- continue
- if len(addrinfo[4]) < 2:
- continue
- if not isinstance(addrinfo[4][0], basestring):
- continue
- except TypeError:
- continue
-
- valid_addrinfos.append(addrinfo)
-
- return valid_addrinfos
diff --git a/pym/_emerge/sync/old_tree_timestamp.py b/pym/_emerge/sync/old_tree_timestamp.py
deleted file mode 100644
index aa23a27..0000000
--- a/pym/_emerge/sync/old_tree_timestamp.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# Copyright 2010-2014 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-from __future__ import division
-
-import locale
-import logging
-import time
-
-from portage import os
-from portage.exception import PortageException
-from portage.localization import _
-from portage.output import EOutput
-from portage.util import grabfile, writemsg_level
-
-def have_english_locale():
- lang, enc = locale.getdefaultlocale()
- if lang is not None:
- lang = lang.lower()
- lang = lang.split('_', 1)[0]
- return lang is None or lang in ('c', 'en')
-
-def whenago(seconds):
- sec = int(seconds)
- mins = 0
- days = 0
- hrs = 0
- years = 0
- out = []
-
- if sec > 60:
- mins = sec // 60
- sec = sec % 60
- if mins > 60:
- hrs = mins // 60
- mins = mins % 60
- if hrs > 24:
- days = hrs // 24
- hrs = hrs % 24
- if days > 365:
- years = days // 365
- days = days % 365
-
- if years:
- out.append("%dy " % years)
- if days:
- out.append("%dd " % days)
- if hrs:
- out.append("%dh " % hrs)
- if mins:
- out.append("%dm " % mins)
- if sec:
- out.append("%ds " % sec)
-
- return "".join(out).strip()
-
-def old_tree_timestamp_warn(portdir, settings):
- unixtime = time.time()
- default_warnsync = 30
-
- timestamp_file = os.path.join(portdir, "metadata/timestamp.x")
- try:
- lastsync = grabfile(timestamp_file)
- except PortageException:
- return False
-
- if not lastsync:
- return False
-
- lastsync = lastsync[0].split()
- if not lastsync:
- return False
-
- try:
- lastsync = int(lastsync[0])
- except ValueError:
- return False
-
- var_name = 'PORTAGE_SYNC_STALE'
- try:
- warnsync = float(settings.get(var_name, default_warnsync))
- except ValueError:
- writemsg_level("!!! %s contains non-numeric value: %s\n" % \
- (var_name, settings[var_name]),
- level=logging.ERROR, noiselevel=-1)
- return False
-
- if warnsync <= 0:
- return False
-
- if (unixtime - 86400 * warnsync) > lastsync:
- out = EOutput()
- if have_english_locale():
- out.ewarn("Last emerge --sync was %s ago." % \
- whenago(unixtime - lastsync))
- else:
- out.ewarn(_("Last emerge --sync was %s.") % \
- time.strftime('%c', time.localtime(lastsync)))
- return True
- return False
diff --git a/pym/portage/emaint/main.py b/pym/portage/emaint/main.py
index 646883d..fea4832 100644
--- a/pym/portage/emaint/main.py
+++ b/pym/portage/emaint/main.py
@@ -9,8 +9,8 @@ import textwrap
import portage
from portage import os
-from portage.emaint.module import Modules
-from portage.emaint.progress import ProgressBar
+from portage.module import Modules
+from portage.progress import ProgressBar
from portage.emaint.defaults import DEFAULT_OPTIONS
from portage.util._argparse import ArgumentParser
@@ -113,7 +113,7 @@ class TaskHandler(object):
for task in tasks:
inst = task()
show_progress = self.show_progress_bar and self.isatty
- # check if the function is capable of progressbar
+ # check if the function is capable of progressbar
# and possibly override it off
if show_progress and hasattr(inst, 'can_progressbar'):
show_progress = inst.can_progressbar(func)
@@ -153,7 +153,13 @@ def emaint_main(myargv):
# files (such as the world file) have sane permissions.
os.umask(0o22)
- module_controller = Modules(namepath="portage.emaint.modules")
+ module_path = os.path.join(
+ (os.path.dirname(
+ os.path.realpath(__file__))), "modules"
+ )
+ module_controller = Modules(
+ path=module_path,
+ namepath="portage.emaint.modules")
module_names = module_controller.module_names[:]
module_names.insert(0, "all")
@@ -206,9 +212,9 @@ def emaint_main(myargv):
tasks = []
for m in module_names[1:]:
#print("DEBUG: module: %s, functions: " % (m, str(module_controller.get_functions(m))))
- if func in module_controller.get_functions(m):
+ if long_action in module_controller.get_functions(m):
tasks.append(module_controller.get_class(m))
- elif func in module_controller.get_functions(args[0]):
+ elif long_action in module_controller.get_functions(args[0]):
tasks = [module_controller.get_class(args[0] )]
else:
portage.util.writemsg(
@@ -221,5 +227,6 @@ def emaint_main(myargv):
# need to pass the parser options dict to the modules
# so they are available if needed.
task_opts = options.__dict__
+ task_opts['return-messages'] = True
taskmaster = TaskHandler(callback=print_results, module_output=sys.stdout)
taskmaster.run_tasks(tasks, func, status, options=task_opts)
diff --git a/pym/portage/emaint/module.py b/pym/portage/emaint/module.py
deleted file mode 100644
index 07a0cb7..0000000
--- a/pym/portage/emaint/module.py
+++ /dev/null
@@ -1,194 +0,0 @@
-# Copyright 2005-2014 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-
-from __future__ import print_function
-
-from portage import os
-from portage.exception import PortageException
-from portage.cache.mappings import ProtectedDict
-
-
-class InvalidModuleName(PortageException):
- """An invalid or unknown module name."""
-
-
-class Module(object):
- """Class to define and hold our plug-in module
-
- @type name: string
- @param name: the module name
- @type path: the path to the new module
- """
-
- def __init__(self, name, namepath):
- """Some variables initialization"""
- self.name = name
- self._namepath = namepath
- self.kids_names = []
- self.kids = {}
- self.initialized = self._initialize()
-
- def _initialize(self):
- """Initialize the plug-in module
-
- @rtype: boolean
- """
- self.valid = False
- try:
- mod_name = ".".join([self._namepath, self.name])
- self._module = __import__(mod_name, [], [], ["not empty"])
- self.valid = True
- except ImportError as e:
- print("MODULE; failed import", mod_name, " error was:", e)
- return False
- self.module_spec = self._module.module_spec
- for submodule in self.module_spec['provides']:
- kid = self.module_spec['provides'][submodule]
- kidname = kid['name']
- kid['module_name'] = '.'.join([mod_name, self.name])
- kid['is_imported'] = False
- self.kids[kidname] = kid
- self.kids_names.append(kidname)
- return True
-
- def get_class(self, name):
- if not name or name not in self.kids_names:
- raise InvalidModuleName("Module name '%s' was invalid or not"
- %name + "part of the module '%s'" %self.name)
- kid = self.kids[name]
- if kid['is_imported']:
- module = kid['instance']
- else:
- try:
- module = __import__(kid['module_name'], [], [], ["not empty"])
- kid['instance'] = module
- kid['is_imported'] = True
- except ImportError:
- raise
- mod_class = getattr(module, kid['class'])
- return mod_class
-
-
-class Modules(object):
- """Dynamic modules system for loading and retrieving any of the
- installed emaint modules and/or provided class's
-
- @param path: Optional path to the "modules" directory or
- defaults to the directory of this file + '/modules'
- @param namepath: Optional python import path to the "modules" directory or
- defaults to the directory name of this file + '.modules'
- """
-
- def __init__(self, path=None, namepath=None):
- if path:
- self._module_path = path
- else:
- self._module_path = os.path.join((
- os.path.dirname(os.path.realpath(__file__))), "modules")
- if namepath:
- self._namepath = namepath
- else:
- self._namepath = '.'.join(os.path.dirname(
- os.path.realpath(__file__)), "modules")
- self._modules = self._get_all_modules()
- self.modules = ProtectedDict(self._modules)
- self.module_names = sorted(self._modules)
- #self.modules = {}
- #for mod in self.module_names:
- #self.module[mod] = LazyLoad(
-
- def _get_all_modules(self):
- """scans the emaint modules dir for loadable modules
-
- @rtype: dictionary of module_plugins
- """
- module_dir = self._module_path
- importables = []
- names = os.listdir(module_dir)
- for entry in names:
- # skip any __init__ or __pycache__ files or directories
- if entry.startswith('__'):
- continue
- try:
- # test for statinfo to ensure it should a real module
- # it will bail if it errors
- os.lstat(os.path.join(module_dir, entry, '__init__.py'))
- importables.append(entry)
- except EnvironmentError:
- pass
- kids = {}
- for entry in importables:
- new_module = Module(entry, self._namepath)
- for module_name in new_module.kids:
- kid = new_module.kids[module_name]
- kid['parent'] = new_module
- kids[kid['name']] = kid
- return kids
-
- def get_module_names(self):
- """Convienence function to return the list of installed modules
- available
-
- @rtype: list
- @return: the installed module names available
- """
- return self.module_names
-
- def get_class(self, modname):
- """Retrieves a module class desired
-
- @type modname: string
- @param modname: the module class name
- """
- if modname and modname in self.module_names:
- mod = self._modules[modname]['parent'].get_class(modname)
- else:
- raise InvalidModuleName("Module name '%s' was invalid or not"
- %modname + "found")
- return mod
-
- def get_description(self, modname):
- """Retrieves the module class decription
-
- @type modname: string
- @param modname: the module class name
- @type string
- @return: the modules class decription
- """
- if modname and modname in self.module_names:
- mod = self._modules[modname]['description']
- else:
- raise InvalidModuleName("Module name '%s' was invalid or not"
- %modname + "found")
- return mod
-
- def get_functions(self, modname):
- """Retrieves the module class exported function names
-
- @type modname: string
- @param modname: the module class name
- @type list
- @return: the modules class exported function names
- """
- if modname and modname in self.module_names:
- mod = self._modules[modname]['functions']
- else:
- raise InvalidModuleName("Module name '%s' was invalid or not"
- %modname + "found")
- return mod
-
- def get_func_descriptions(self, modname):
- """Retrieves the module class exported functions descriptions
-
- @type modname: string
- @param modname: the module class name
- @type dictionary
- @return: the modules class exported functions descriptions
- """
- if modname and modname in self.module_names:
- desc = self._modules[modname]['func_desc']
- else:
- raise InvalidModuleName("Module name '%s' was invalid or not"
- %modname + "found")
- return desc
diff --git a/pym/portage/emaint/modules/sync/__init__.py b/pym/portage/emaint/modules/sync/__init__.py
new file mode 100644
index 0000000..4070200
--- /dev/null
+++ b/pym/portage/emaint/modules/sync/__init__.py
@@ -0,0 +1,44 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+"""Check repos.conf settings and sync repositories.
+"""
+
+
+module_spec = {
+ 'name': 'sync',
+ 'description': __doc__,
+ 'provides':{
+ 'sync-module': {
+ 'name': "sync",
+ 'class': "SyncRepos",
+ 'description': __doc__,
+ 'functions': ['allrepos', 'auto', 'repo'],
+ 'func_desc': {
+ 'repo': {
+ "short": "-r", "long": "--repo",
+ "help": "(sync module only): -r, --repo Sync the specified repo",
+ 'status': "Syncing %s",
+ 'action': 'store',
+ 'func': 'repo',
+ },
+ 'allrepos': {
+ "short": "-A", "long": "--allrepos",
+ "help": "(sync module only): -A, --allrepos Sync all repos that have a sync-url defined",
+ 'status': "Syncing %s",
+ 'action': 'store_true',
+ 'dest': 'allrepos',
+ 'func': 'all_repos',
+ },
+ 'auto': {
+ "short": "-a", "long": "--auto",
+ "help": "(sync module only): -a, --auto Sync auto-sync enabled repos only",
+ 'status': "Syncing %s",
+ 'action': 'store_true',
+ 'dest': 'auto',
+ 'func': 'auto_sync',
+ },
+ }
+ }
+ }
+ }
diff --git a/pym/portage/emaint/modules/sync/sync.py b/pym/portage/emaint/modules/sync/sync.py
new file mode 100644
index 0000000..ab7591d
--- /dev/null
+++ b/pym/portage/emaint/modules/sync/sync.py
@@ -0,0 +1,238 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import os
+import sys
+
+import portage
+portage._internal_caller = True
+portage._sync_mode = True
+from portage.localization import _
+from portage.output import bold, create_color_func
+from portage.sync import get_syncer
+from portage._global_updates import _global_updates
+from portage.util import writemsg_level
+
+import _emerge
+from _emerge.emergelog import emergelog
+
+
+portage.proxy.lazyimport.lazyimport(globals(),
+ '_emerge.actions:adjust_configs,load_emerge_config',
+ '_emerge.chk_updated_cfg_files:chk_updated_cfg_files',
+ '_emerge.main:parse_opts',
+ '_emerge.post_emerge:display_news_notification',
+)
+
+warn = create_color_func("WARN")
+
+if sys.hexversion >= 0x3000000:
+ _basestring = str
+else:
+ _basestring = basestring
+
+
+class SyncRepos(object):
+
+ short_desc = "Check repos.conf settings and/or sync repositories"
+
+ @staticmethod
+ def name():
+ return "sync"
+
+
+ def can_progressbar(self, func):
+ return False
+
+
+ def __init__(self, emerge_config=None, emerge_logging=False):
+ '''Class init function
+
+ @param emerge_config: optional an emerge_config instance to use
+ @param emerge_logging: boolean, defaults to False
+ '''
+ if emerge_config:
+ self.emerge_config = emerge_config
+ else:
+ # need a basic options instance
+ actions, opts, _files = parse_opts([], silent=True)
+ self.emerge_config = load_emerge_config(
+ action='sync', args=_files, trees=[], opts=opts)
+ if emerge_logging:
+ _emerge.emergelog._disable = False
+ self.xterm_titles = "notitles" not in \
+ self.emerge_config.target_config.settings.features
+ emergelog(self.xterm_titles, " === sync")
+
+
+ def auto_sync(self, **kwargs):
+ '''Sync auto-sync enabled repos'''
+ options = kwargs.get('options', None)
+ selected = self._get_repos(True)
+ if options.get('return-messages', False):
+ return self.rmessage(self._sync(selected), 'sync')
+ return self._sync(selected)
+
+
+ def all_repos(self, **kwargs):
+ '''Sync all repos defined in repos.conf'''
+ selected = self._get_repos(auto_sync_only=False)
+ options = kwargs.get('options', None)
+ if options.get('return-messages', False):
+ return self.rmessage(
+ self._sync(selected),
+ 'sync')
+ return self._sync(selected)
+
+
+ def repo(self, **kwargs):
+ '''Sync the specified repo'''
+ options = kwargs.get('options', None)
+ if options:
+ repos = options.get('repo', '')
+ return_messages = options.get('return-messages', False)
+ else:
+ return_messages = False
+ if isinstance(repos, _basestring):
+ repos = repos.split()
+ available = self._get_repos(auto_sync_only=False)
+ selected = self._match_repos(repos, available)
+ if return_messages:
+ return self.rmessage(self._sync(selected), 'sync')
+ return self._sync(selected)
+
+
+ @staticmethod
+ def _match_repos(repos, available):
+ '''Internal search, matches up the repo.name in repos
+
+ @param repos: list, of repo names to match
+ @param avalable: list of repo objects to search
+ @return: list of repo objects that match
+ '''
+ selected = []
+ for repo in available:
+ if repo.name in repos:
+ selected.append(repo)
+ return selected
+
+
+ def _get_repos(self, auto_sync_only=True):
+ selected_repos = []
+ unknown_repo_names = []
+ missing_sync_type = []
+ if self.emerge_config.args:
+ for repo_name in self.emerge_config.args:
+ print("_get_repos(): repo_name =", repo_name)
+ try:
+ repo = self.emerge_config.target_config.settings.repositories[repo_name]
+ except KeyError:
+ unknown_repo_names.append(repo_name)
+ else:
+ selected_repos.append(repo)
+ if repo.sync_type is None:
+ missing_sync_type.append(repo)
+
+ if unknown_repo_names:
+ writemsg_level("!!! %s\n" % _("Unknown repo(s): %s") %
+ " ".join(unknown_repo_names),
+ level=logging.ERROR, noiselevel=-1)
+
+ if missing_sync_type:
+ writemsg_level("!!! %s\n" %
+ _("Missing sync-type for repo(s): %s") %
+ " ".join(repo.name for repo in missing_sync_type),
+ level=logging.ERROR, noiselevel=-1)
+
+ if unknown_repo_names or missing_sync_type:
+ print("missing or unknown repos... returning")
+ return []
+
+ else:
+ selected_repos.extend(self.emerge_config.target_config.settings.repositories)
+ #print("_get_repos(), selected =", selected_repos)
+ if auto_sync_only:
+ return self._filter_auto(selected_repos)
+ return selected_repos
+
+
+ def _filter_auto(self, repos):
+ selected = []
+ for repo in repos:
+ if repo.auto_sync in ['yes', 'true']:
+ selected.append(repo)
+ return selected
+
+
+ def _sync(self, selected_repos):
+ if not selected_repos:
+ print("_sync(), nothing to sync... returning")
+ return [('None', os.EX_OK)]
+ # Portage needs to ensure a sane umask for the files it creates.
+ os.umask(0o22)
+
+ sync_manager = get_syncer(self.emerge_config.target_config.settings, emergelog)
+ retvals = []
+ for repo in selected_repos:
+ print("syncing repo:", repo.name)
+ if repo.sync_type is not None:
+ returncode = sync_manager.sync(self.emerge_config, repo)
+ #if returncode != os.EX_OK:
+ retvals.append((repo.name, returncode))
+
+ # Reload the whole config.
+ portage._sync_mode = False
+ self._reload_config()
+ self._do_pkg_moves()
+ self._check_updates()
+ display_news_notification(self.emerge_config.target_config,
+ self.emerge_config.opts)
+ if retvals:
+ return retvals
+ return [('None', os.EX_OK)]
+
+
+ def _do_pkg_moves(self):
+ if self.emerge_config.opts.get('--package-moves') != 'n' and \
+ _global_updates(self.emerge_config.trees,
+ self.emerge_config.target_config.mtimedb["updates"],
+ quiet=("--quiet" in self.emerge_config.opts)):
+ self.emerge_config.target_config.mtimedb.commit()
+ # Reload the whole config.
+ self._reload_config()
+
+
+ def _check_updates(self):
+ mybestpv = self.emerge_config.target_config.trees['porttree'].dbapi.xmatch(
+ "bestmatch-visible", portage.const.PORTAGE_PACKAGE_ATOM)
+ mypvs = portage.best(
+ self.emerge_config.target_config.trees['vartree'].dbapi.match(
+ portage.const.PORTAGE_PACKAGE_ATOM))
+
+ chk_updated_cfg_files(self.emerge_config.target_config.root,
+ portage.util.shlex_split(
+ self.emerge_config.target_config.settings.get("CONFIG_PROTECT", "")))
+
+ if mybestpv != mypvs and "--quiet" not in self.emerge_config.opts:
+ print()
+ print(warn(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended")
+ print(warn(" * ")+"that you update portage now, before any other packages are updated.")
+ print()
+ print(warn(" * ")+"To update portage, run 'emerge --oneshot portage' now.")
+ print()
+
+
+ def _reload_config(self):
+ '''Reload the whole config from scratch.'''
+ load_emerge_config(emerge_config=self.emerge_config)
+ adjust_configs(self.emerge_config.opts, self.emerge_config.trees)
+
+
+ def rmessage(self, rvals, action):
+ '''Creates emaint style messages to return to the task handler'''
+ messages = []
+ for rval in rvals:
+ messages.append("Action: %s for repo: %s, returned code = %s"
+ % (action, rval[0], rval[1]))
+ return messages
diff --git a/pym/portage/emaint/progress.py b/pym/portage/emaint/progress.py
deleted file mode 100644
index e43c2af..0000000
--- a/pym/portage/emaint/progress.py
+++ /dev/null
@@ -1,61 +0,0 @@
-# Copyright 2005-2012 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-import time
-import signal
-
-import portage
-
-
-class ProgressHandler(object):
- def __init__(self):
- self.reset()
-
- def reset(self):
- self.curval = 0
- self.maxval = 0
- self.last_update = 0
- self.min_display_latency = 0.2
-
- def onProgress(self, maxval, curval):
- self.maxval = maxval
- self.curval = curval
- cur_time = time.time()
- if cur_time - self.last_update >= self.min_display_latency:
- self.last_update = cur_time
- self.display()
-
- def display(self):
- raise NotImplementedError(self)
-
-
-class ProgressBar(ProgressHandler):
- """Class to set up and return a Progress Bar"""
-
- def __init__(self, isatty, **kwargs):
- self.isatty = isatty
- self.kwargs = kwargs
- ProgressHandler.__init__(self)
- self.progressBar = None
-
- def start(self):
- if self.isatty:
- self.progressBar = portage.output.TermProgressBar(**self.kwargs)
- signal.signal(signal.SIGWINCH, self.sigwinch_handler)
- else:
- self.onProgress = None
- return self.onProgress
-
- def set_label(self, _label):
- self.kwargs['label'] = _label
-
- def display(self):
- self.progressBar.set(self.curval, self.maxval)
-
- def sigwinch_handler(self, signum, frame):
- lines, self.progressBar.term_columns = \
- portage.output.get_term_size()
-
- def stop(self):
- signal.signal(signal.SIGWINCH, signal.SIG_DFL)
-
diff --git a/pym/portage/metadata.py b/pym/portage/metadata.py
new file mode 100644
index 0000000..d2dacc7
--- /dev/null
+++ b/pym/portage/metadata.py
@@ -0,0 +1,210 @@
+
+from __future__ import print_function
+
+import sys
+import signal
+import logging
+import operator
+
+import portage
+from portage import os
+from portage import eapi_is_supported
+from portage.util import writemsg_level
+from portage.cache.cache_errors import CacheError
+from _emerge.ProgressHandler import ProgressHandler
+from portage.eclass_cache import hashed_path
+
+
+def action_metadata(settings, portdb, myopts, porttrees=None):
+ if porttrees is None:
+ porttrees = portdb.porttrees
+ portage.writemsg_stdout("\n>>> Updating Portage cache\n")
+ #old_umask = os.umask(0o002)
+ cachedir = os.path.normpath(settings.depcachedir)
+ if cachedir in ["/", "/bin", "/dev", "/etc", "/home",
+ "/lib", "/opt", "/proc", "/root", "/sbin",
+ "/sys", "/tmp", "/usr", "/var"]:
+ print("!!! PORTAGE_DEPCACHEDIR IS SET TO A PRIMARY " + \
+ "ROOT DIRECTORY ON YOUR SYSTEM.", file=sys.stderr)
+ print("!!! This is ALMOST CERTAINLY NOT what you want: '%s'" % cachedir, file=sys.stderr)
+ sys.exit(73)
+ if not os.path.exists(cachedir):
+ os.makedirs(cachedir)
+
+ auxdbkeys = portdb._known_keys
+
+ class TreeData(object):
+ __slots__ = ('dest_db', 'eclass_db', 'path', 'src_db', 'valid_nodes')
+ def __init__(self, dest_db, eclass_db, path, src_db):
+ self.dest_db = dest_db
+ self.eclass_db = eclass_db
+ self.path = path
+ self.src_db = src_db
+ self.valid_nodes = set()
+
+ porttrees_data = []
+ for path in porttrees:
+ src_db = portdb._pregen_auxdb.get(path)
+ if src_db is None:
+ # portdbapi does not populate _pregen_auxdb
+ # when FEATURES=metadata-transfer is enabled
+ src_db = portdb._create_pregen_cache(path)
+
+ if src_db is not None:
+ porttrees_data.append(TreeData(portdb.auxdb[path],
+ portdb.repositories.get_repo_for_location(path).eclass_db, path, src_db))
+
+ porttrees = [tree_data.path for tree_data in porttrees_data]
+
+ quiet = settings.get('TERM') == 'dumb' or \
+ '--quiet' in myopts or \
+ not sys.stdout.isatty()
+
+ onProgress = None
+ if not quiet:
+ progressBar = portage.output.TermProgressBar()
+ progressHandler = ProgressHandler()
+ onProgress = progressHandler.onProgress
+ def display():
+ progressBar.set(progressHandler.curval, progressHandler.maxval)
+ progressHandler.display = display
+ def sigwinch_handler(signum, frame):
+ lines, progressBar.term_columns = \
+ portage.output.get_term_size()
+ signal.signal(signal.SIGWINCH, sigwinch_handler)
+
+ # Temporarily override portdb.porttrees so portdb.cp_all()
+ # will only return the relevant subset.
+ portdb_porttrees = portdb.porttrees
+ portdb.porttrees = porttrees
+ try:
+ cp_all = portdb.cp_all()
+ finally:
+ portdb.porttrees = portdb_porttrees
+
+ curval = 0
+ maxval = len(cp_all)
+ if onProgress is not None:
+ onProgress(maxval, curval)
+
+ # TODO: Display error messages, but do not interfere with the progress bar.
+ # Here's how:
+ # 1) erase the progress bar
+ # 2) show the error message
+ # 3) redraw the progress bar on a new line
+
+ for cp in cp_all:
+ for tree_data in porttrees_data:
+
+ src_chf = tree_data.src_db.validation_chf
+ dest_chf = tree_data.dest_db.validation_chf
+ dest_chf_key = '_%s_' % dest_chf
+ dest_chf_getter = operator.attrgetter(dest_chf)
+
+ for cpv in portdb.cp_list(cp, mytree=tree_data.path):
+ tree_data.valid_nodes.add(cpv)
+ try:
+ src = tree_data.src_db[cpv]
+ except (CacheError, KeyError):
+ continue
+
+ ebuild_location = portdb.findname(cpv, mytree=tree_data.path)
+ if ebuild_location is None:
+ continue
+ ebuild_hash = hashed_path(ebuild_location)
+
+ try:
+ if not tree_data.src_db.validate_entry(src,
+ ebuild_hash, tree_data.eclass_db):
+ continue
+ except CacheError:
+ continue
+
+ eapi = src.get('EAPI')
+ if not eapi:
+ eapi = '0'
+ eapi_supported = eapi_is_supported(eapi)
+ if not eapi_supported:
+ continue
+
+ dest = None
+ try:
+ dest = tree_data.dest_db[cpv]
+ except (KeyError, CacheError):
+ pass
+
+ for d in (src, dest):
+ if d is not None and d.get('EAPI') in ('', '0'):
+ del d['EAPI']
+
+ if src_chf != 'mtime':
+ # src may contain an irrelevant _mtime_ which corresponds
+ # to the time that the cache entry was written
+ src.pop('_mtime_', None)
+
+ if src_chf != dest_chf:
+ # populate src entry with dest_chf_key
+ # (the validity of the dest_chf that we generate from the
+ # ebuild here relies on the fact that we already used
+ # validate_entry to validate the ebuild with src_chf)
+ src[dest_chf_key] = dest_chf_getter(ebuild_hash)
+
+ if dest is not None:
+ if not (dest[dest_chf_key] == src[dest_chf_key] and \
+ tree_data.eclass_db.validate_and_rewrite_cache(
+ dest['_eclasses_'], tree_data.dest_db.validation_chf,
+ tree_data.dest_db.store_eclass_paths) is not None and \
+ set(dest['_eclasses_']) == set(src['_eclasses_'])):
+ dest = None
+ else:
+ # We don't want to skip the write unless we're really
+ # sure that the existing cache is identical, so don't
+ # trust _mtime_ and _eclasses_ alone.
+ for k in auxdbkeys:
+ if dest.get(k, '') != src.get(k, ''):
+ dest = None
+ break
+
+ if dest is not None:
+ # The existing data is valid and identical,
+ # so there's no need to overwrite it.
+ continue
+
+ try:
+ tree_data.dest_db[cpv] = src
+ except CacheError:
+ # ignore it; can't do anything about it.
+ pass
+
+ curval += 1
+ if onProgress is not None:
+ onProgress(maxval, curval)
+
+ if onProgress is not None:
+ onProgress(maxval, curval)
+
+ for tree_data in porttrees_data:
+ try:
+ dead_nodes = set(tree_data.dest_db)
+ except CacheError as e:
+ writemsg_level("Error listing cache entries for " + \
+ "'%s': %s, continuing...\n" % (tree_data.path, e),
+ level=logging.ERROR, noiselevel=-1)
+ del e
+ else:
+ dead_nodes.difference_update(tree_data.valid_nodes)
+ for cpv in dead_nodes:
+ try:
+ del tree_data.dest_db[cpv]
+ except (KeyError, CacheError):
+ pass
+
+ if not quiet:
+ # make sure the final progress is displayed
+ progressHandler.display()
+ print()
+ signal.signal(signal.SIGWINCH, signal.SIG_DFL)
+
+ portdb.flush_cache()
+ sys.stdout.flush()
+ os.umask(old_umask)
diff --git a/pym/portage/module.py b/pym/portage/module.py
new file mode 100644
index 0000000..d961574
--- /dev/null
+++ b/pym/portage/module.py
@@ -0,0 +1,186 @@
+# Copyright 2005-2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+
+from __future__ import print_function
+
+from portage import os
+from portage.exception import PortageException
+from portage.cache.mappings import ProtectedDict
+
+
+class InvalidModuleName(PortageException):
+ """An invalid or unknown module name."""
+
+
+class Module(object):
+ """Class to define and hold our plug-in module
+
+ @type name: string
+ @param name: the module name
+ @type path: the path to the new module
+ """
+
+ def __init__(self, name, namepath):
+ """Some variables initialization"""
+ self.name = name
+ self._namepath = namepath
+ self.kids_names = []
+ self.kids = {}
+ self.initialized = self._initialize()
+
+ def _initialize(self):
+ """Initialize the plug-in module
+
+ @rtype: boolean
+ """
+ self.valid = False
+ try:
+ mod_name = ".".join([self._namepath, self.name])
+ self._module = __import__(mod_name, [], [], ["not empty"])
+ self.valid = True
+ except ImportError as e:
+ print("MODULE; failed import", mod_name, " error was:", e)
+ return False
+ self.module_spec = self._module.module_spec
+ for submodule in self.module_spec['provides']:
+ kid = self.module_spec['provides'][submodule]
+ kidname = kid['name']
+ kid['module_name'] = '.'.join([mod_name, self.name])
+ kid['is_imported'] = False
+ self.kids[kidname] = kid
+ self.kids_names.append(kidname)
+ return True
+
+ def get_class(self, name):
+ if not name or name not in self.kids_names:
+ raise InvalidModuleName("Module name '%s' was invalid or not"
+ %name + "part of the module '%s'" %self.name)
+ kid = self.kids[name]
+ if kid['is_imported']:
+ module = kid['instance']
+ else:
+ try:
+ module = __import__(kid['module_name'], [], [], ["not empty"])
+ kid['instance'] = module
+ kid['is_imported'] = True
+ except ImportError:
+ raise
+ mod_class = getattr(module, kid['class'])
+ return mod_class
+
+
+class Modules(object):
+ """Dynamic modules system for loading and retrieving any of the
+ installed emaint modules and/or provided class's
+
+ @param path: Optional path to the "modules" directory or
+ defaults to the directory of this file + '/modules'
+ @param namepath: Optional python import path to the "modules" directory or
+ defaults to the directory name of this file + '.modules'
+ """
+
+ def __init__(self, path, namepath):
+ self._module_path = path
+ self._namepath = namepath
+ self._modules = self._get_all_modules()
+ self.modules = ProtectedDict(self._modules)
+ self.module_names = sorted(self._modules)
+ #self.modules = {}
+ #for mod in self.module_names:
+ #self.module[mod] = LazyLoad(
+
+ def _get_all_modules(self):
+ """scans the emaint modules dir for loadable modules
+
+ @rtype: dictionary of module_plugins
+ """
+ module_dir = self._module_path
+ importables = []
+ names = os.listdir(module_dir)
+ for entry in names:
+ # skip any __init__ or __pycache__ files or directories
+ if entry.startswith('__'):
+ continue
+ try:
+ # test for statinfo to ensure it should a real module
+ # it will bail if it errors
+ os.lstat(os.path.join(module_dir, entry, '__init__.py'))
+ importables.append(entry)
+ except EnvironmentError:
+ pass
+ kids = {}
+ for entry in importables:
+ new_module = Module(entry, self._namepath)
+ for module_name in new_module.kids:
+ kid = new_module.kids[module_name]
+ kid['parent'] = new_module
+ kids[kid['name']] = kid
+ return kids
+
+ def get_module_names(self):
+ """Convienence function to return the list of installed modules
+ available
+
+ @rtype: list
+ @return: the installed module names available
+ """
+ return self.module_names
+
+ def get_class(self, modname):
+ """Retrieves a module class desired
+
+ @type modname: string
+ @param modname: the module class name
+ """
+ if modname and modname in self.module_names:
+ mod = self._modules[modname]['parent'].get_class(modname)
+ else:
+ raise InvalidModuleName("Module name '%s' was invalid or not"
+ %modname + "found")
+ return mod
+
+ def get_description(self, modname):
+ """Retrieves the module class decription
+
+ @type modname: string
+ @param modname: the module class name
+ @type string
+ @return: the modules class decription
+ """
+ if modname and modname in self.module_names:
+ mod = self._modules[modname]['description']
+ else:
+ raise InvalidModuleName("Module name '%s' was invalid or not"
+ %modname + "found")
+ return mod
+
+ def get_functions(self, modname):
+ """Retrieves the module class exported function names
+
+ @type modname: string
+ @param modname: the module class name
+ @type list
+ @return: the modules class exported function names
+ """
+ if modname and modname in self.module_names:
+ mod = self._modules[modname]['functions']
+ else:
+ raise InvalidModuleName("Module name '%s' was invalid or not"
+ %modname + "found")
+ return mod
+
+ def get_func_descriptions(self, modname):
+ """Retrieves the module class exported functions descriptions
+
+ @type modname: string
+ @param modname: the module class name
+ @type dictionary
+ @return: the modules class exported functions descriptions
+ """
+ if modname and modname in self.module_names:
+ desc = self._modules[modname]['func_desc']
+ else:
+ raise InvalidModuleName("Module name '%s' was invalid or not"
+ %modname + "found")
+ return desc
diff --git a/pym/portage/progress.py b/pym/portage/progress.py
new file mode 100644
index 0000000..e43c2af
--- /dev/null
+++ b/pym/portage/progress.py
@@ -0,0 +1,61 @@
+# Copyright 2005-2012 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import time
+import signal
+
+import portage
+
+
+class ProgressHandler(object):
+ def __init__(self):
+ self.reset()
+
+ def reset(self):
+ self.curval = 0
+ self.maxval = 0
+ self.last_update = 0
+ self.min_display_latency = 0.2
+
+ def onProgress(self, maxval, curval):
+ self.maxval = maxval
+ self.curval = curval
+ cur_time = time.time()
+ if cur_time - self.last_update >= self.min_display_latency:
+ self.last_update = cur_time
+ self.display()
+
+ def display(self):
+ raise NotImplementedError(self)
+
+
+class ProgressBar(ProgressHandler):
+ """Class to set up and return a Progress Bar"""
+
+ def __init__(self, isatty, **kwargs):
+ self.isatty = isatty
+ self.kwargs = kwargs
+ ProgressHandler.__init__(self)
+ self.progressBar = None
+
+ def start(self):
+ if self.isatty:
+ self.progressBar = portage.output.TermProgressBar(**self.kwargs)
+ signal.signal(signal.SIGWINCH, self.sigwinch_handler)
+ else:
+ self.onProgress = None
+ return self.onProgress
+
+ def set_label(self, _label):
+ self.kwargs['label'] = _label
+
+ def display(self):
+ self.progressBar.set(self.curval, self.maxval)
+
+ def sigwinch_handler(self, signum, frame):
+ lines, self.progressBar.term_columns = \
+ portage.output.get_term_size()
+
+ def stop(self):
+ signal.signal(signal.SIGWINCH, signal.SIG_DFL)
+
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 5e0d055..10bd477 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -31,6 +31,7 @@ from portage import _unicode_decode
from portage import _unicode_encode
from portage import _encodings
from portage import manifest
+import portage.sync
if sys.hexversion >= 0x3000000:
# pylint: disable=W0622
@@ -78,8 +79,8 @@ class RepoConfig(object):
"""Stores config of one repository"""
__slots__ = ('aliases', 'allow_missing_manifest', 'allow_provide_virtual',
- 'cache_formats', 'create_manifest', 'disable_manifest', 'eapi',
- 'eclass_db', 'eclass_locations', 'eclass_overrides',
+ 'auto_sync', 'cache_formats', 'create_manifest', 'disable_manifest',
+ 'eapi', 'eclass_db', 'eclass_locations', 'eclass_overrides',
'find_invalid_path_char', 'force', 'format', 'local_config', 'location',
'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name',
'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority',
@@ -158,6 +159,11 @@ class RepoConfig(object):
sync_uri = sync_uri.strip()
self.sync_uri = sync_uri or None
+ auto_sync = repo_opts.get('auto-sync')
+ if auto_sync is not None:
+ auto_sync = auto_sync.strip().lower()
+ self.auto_sync = auto_sync
+
# Not implemented.
format = repo_opts.get('format')
if format is not None:
@@ -456,8 +462,10 @@ class RepoConfigLoader(object):
if repos_conf_opts is not None:
# Selectively copy only the attributes which
# repos.conf is allowed to override.
- for k in ('aliases', 'eclass_overrides', 'force', 'masters',
- 'priority', 'sync_cvs_repo', 'sync_type', 'sync_uri'):
+ for k in ('aliases', 'auto_sync', 'eclass_overrides',
+ 'force', 'masters', 'priority', 'sync_cvs_repo',
+ 'sync_type', 'sync_uri',
+ ):
v = getattr(repos_conf_opts, k, None)
if v is not None:
setattr(repo, k, v)
@@ -550,25 +558,8 @@ class RepoConfigLoader(object):
repo = RepoConfig(sname, optdict, local_config=local_config)
- if repo.sync_type is not None and repo.sync_uri is None:
- writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute, but is missing sync-uri attribute") %
- sname, level=logging.ERROR, noiselevel=-1)
- continue
-
- if repo.sync_uri is not None and repo.sync_type is None:
- writemsg_level("!!! %s\n" % _("Repository '%s' has sync-uri attribute, but is missing sync-type attribute") %
- sname, level=logging.ERROR, noiselevel=-1)
- continue
-
- if repo.sync_type not in (None, "cvs", "git", "rsync"):
- writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute set to unsupported value: '%s'") %
- (sname, repo.sync_type), level=logging.ERROR, noiselevel=-1)
- continue
-
- if repo.sync_type == "cvs" and repo.sync_cvs_repo is None:
- writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type=cvs, but is missing sync-cvs-repo attribute") %
- sname, level=logging.ERROR, noiselevel=-1)
- continue
+ # Perform repos.conf sync variable validation
+ portage.sync.validate_config(repo, logging)
# For backward compatibility with locations set via PORTDIR and
# PORTDIR_OVERLAY, delay validation of the location and repo.name
diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
new file mode 100644
index 0000000..58a1298
--- /dev/null
+++ b/pym/portage/sync/__init__.py
@@ -0,0 +1,54 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+
+from portage.module import Modules
+from portage.sync.controller import SyncManager
+from portage.sync.config_checks import check_type
+
+sync_manager = None
+
+path = os.path.join(os.path.dirname(__file__), "modules")
+# initial development debug info
+#print("module path:", path)
+
+module_controller = Modules(path=path, namepath="portage.sync.modules")
+
+# initial development debug info
+#print(module_controller.module_names)
+module_names = module_controller.module_names[:]
+
+
+def get_syncer(settings=None, logger=None):
+ '''Initializes and returns the SyncManager instance
+ to be used for sync operations
+
+ @param settings: emerge.settings instance
+ @param logger: emerge logger instance
+ @returns SyncManager instance
+ '''
+ global sync_manager
+ if sync_manager and not settings and not logger:
+ return sync_manager
+ if settings is None:
+ from _emerge.actions import load_emerge_config
+ emerge_config = load_emerge_config()
+ settings = emerge_config.target_config.settings
+ if logger is None:
+ from _emerge.emergelog import emergelog as logger
+ sync_manager = SyncManager(settings, logger)
+ return sync_manager
+
+
+def validate_config(repo, logger):
+ '''Validate the repos.conf settings for the repo'''
+ global module_names, module_controller
+ if not check_type(repo, logger, module_names):
+ return False
+
+ #print(repo)
+ if repo.sync_type:
+ validated = module_controller.modules[repo.sync_type]['validate_config']
+ return validated(repo, logger).repo_checks()
+ return True
diff --git a/pym/portage/sync/config_checks.py b/pym/portage/sync/config_checks.py
new file mode 100644
index 0000000..db316aa
--- /dev/null
+++ b/pym/portage/sync/config_checks.py
@@ -0,0 +1,72 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+'''
+Base class for performing repos.conf sync variables checks.
+This class contains common checks code and functions.
+
+For additional checks or other customizations,
+subclass it adding and/or overriding classes as needed.
+'''
+
+import logging
+
+from portage.localization import _
+from portage.util import writemsg_level
+
+
+def check_type(repo, logger, module_names):
+ if repo.sync_uri is not None and repo.sync_type is None:
+ writemsg_level("!!! %s\n" %
+ _("Repository '%s' has sync-uri attribute, but is missing sync-type attribute")
+ % repo.name, level=logger.ERROR, noiselevel=-1)
+ return False
+ if repo.sync_type not in module_names + [None]:
+ writemsg_level("!!! %s\n" %
+ _("Repository '%s' has sync-type attribute set to unsupported value: '%s'")
+ % (repo.name, repo.sync_type),
+ level=logger.ERROR, noiselevel=-1)
+ writemsg_level("!!! %s\n" %
+ _("Installed sync-types are: '%s'")
+ % (str(module_names)),
+ level=logger.ERROR, noiselevel=-1)
+ return False
+ return True
+
+
+class CheckSyncConfig(object):
+ '''Base repos.conf settings checks class'''
+
+ def __init__(self, repo=None, logger=None):
+ '''Class init function
+
+ @param logger: optional logging instance,
+ defaults to logging module
+ '''
+ self.logger = logger or logging
+ self.repo = repo
+ self.checks = ['check_uri', 'check_auto_sync']
+
+
+ def repo_checks(self):
+ '''Perform all checks available'''
+ for check in self.checks:
+ getattr(self, check)()
+
+
+ def check_uri(self):
+ '''Check the sync_uri setting'''
+ if self.repo.sync_uri is None:
+ writemsg_level("!!! %s\n" % _("Repository '%s' has sync-type attribute, but is missing sync-uri attribute")
+ % self.repo.name, level=self.logger.ERROR, noiselevel=-1)
+
+
+ def check_auto_sync(self):
+ '''Check the auto_sync setting'''
+ if self.repo.auto_sync is None:
+ writemsg_level("!!! %s\n" % _("Repository '%s' is missing auto_sync attribute")
+ % self.repo.name, level=self.logger.ERROR, noiselevel=-1)
+ elif self.repo.auto_sync.lower() not in ["yes", "true", "no", "false"]:
+ writemsg_level("!!! %s\n" % _("Repository '%s' auto_sync attribute must be one of: %s")
+ % (self.repo.name, '{yes, true, no, false}'),
+ level=self.logger.ERROR, noiselevel=-1)
diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
new file mode 100644
index 0000000..0fe723b
--- /dev/null
+++ b/pym/portage/sync/controller.py
@@ -0,0 +1,220 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+
+
+import sys
+import logging
+import pwd
+
+import portage
+from portage import os
+from portage.progress import ProgressBar
+#from portage.emaint.defaults import DEFAULT_OPTIONS
+#from portage.util._argparse import ArgumentParser
+from portage.util import writemsg_level
+from portage.output import create_color_func
+good = create_color_func("GOOD")
+bad = create_color_func("BAD")
+warn = create_color_func("WARN")
+from portage.package.ebuild.doebuild import _check_temp_dir
+from portage.metadata import action_metadata
+
+
+class TaskHandler(object):
+ """Handles the running of the tasks it is given
+ """
+
+ def __init__(self, show_progress_bar=True, verbose=True, callback=None):
+ self.show_progress_bar = show_progress_bar
+ self.verbose = verbose
+ self.callback = callback
+ self.isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty()
+ self.progress_bar = ProgressBar(self.isatty, title="Portage-Sync", max_desc_length=27)
+
+
+ def run_tasks(self, tasks, func, status=None, verbose=True, options=None):
+ """Runs the module tasks"""
+ if tasks is None or func is None:
+ return
+ for task in tasks:
+ inst = task()
+ show_progress = self.show_progress_bar and self.isatty
+ # check if the function is capable of progressbar
+ # and possibly override it off
+ if show_progress and hasattr(inst, 'can_progressbar'):
+ show_progress = inst.can_progressbar(func)
+ if show_progress:
+ self.progress_bar.reset()
+ self.progress_bar.set_label(func + " " + inst.name())
+ onProgress = self.progress_bar.start()
+ else:
+ onProgress = None
+ kwargs = {
+ 'onProgress': onProgress,
+ # pass in a copy of the options so a module can not pollute or change
+ # them for other tasks if there is more to do.
+ 'options': options.copy()
+ }
+ result = getattr(inst, func)(**kwargs)
+ if show_progress:
+ # make sure the final progress is displayed
+ self.progress_bar.display()
+ print()
+ self.progress_bar.stop()
+ if self.callback:
+ self.callback(result)
+
+
+def print_results(results):
+ if results:
+ print()
+ print("\n".join(results))
+ print("\n")
+
+
+class SyncManager(object):
+ '''Main sync control module'''
+
+ def __init__(self, settings, logger):
+ self.logger = logger
+ # Similar to emerge, sync needs a default umask so that created
+ # files have sane permissions.
+ os.umask(0o22)
+
+ self.module_controller = portage.sync.module_controller
+ self.module_names = self.module_controller.module_names[:]
+
+
+ def get_module_descriptions(self, mod):
+ desc = self.module_controller.get_func_descriptions(mod)
+ if desc:
+ return desc
+ return []
+
+
+ def sync(self, emerge_config=None, repo=None, callback=None):
+ self.emerge_config = emerge_config
+ self.callback = callback or self._sync_callback
+ self.repo = repo
+ self.exitcode = 1
+ if repo.sync_type in self.module_names[1:]:
+ tasks = [self.module_controller.get_class(repo.sync_type)]
+ else:
+ portage.util.writemsg(
+ "\nERROR: Sync module '%s' is not an installed/known type'\n\n"
+ % (repo.sync_type), noiselevel=-1)
+ return self.exitcode
+
+ rval = self.pre_sync(repo)
+ if rval != os.EX_OK:
+ return rval
+
+ # need to pass the kwargs dict to the modules
+ # so they are available if needed.
+ task_opts = {
+ 'emerge_config': emerge_config,
+ 'logger': self.logger,
+ 'portdb': self.trees[self.settings['EROOT']]['porttree'].dbapi,
+ 'repo': repo,
+ 'settings': self.settings,
+ 'spawn_kwargs': self.spawn_kwargs,
+ 'usersync_uid': self.usersync_uid,
+ 'xterm_titles': self.xterm_titles,
+ }
+ func = 'sync'
+ status = None
+ taskmaster = TaskHandler(callback=self.do_callback)
+ taskmaster.run_tasks(tasks, func, status, options=task_opts)
+
+ self.perform_post_sync_hook(repo.sync_uri)
+
+ return self.exitcode
+
+
+ def do_callback(self, result):
+ #print("result:", result, "callback()", self.callback)
+ exitcode, updatecache_flg = result
+ if self.callback:
+ self.callback(exitcode, updatecache_flg)
+ return
+
+
+ def perform_post_sync_hook(self, dosyncuri):
+ postsync = os.path.join(self.settings["PORTAGE_CONFIGROOT"],
+ portage.USER_CONFIG_PATH, "bin", "post_sync")
+ if os.access(postsync, os.X_OK):
+ retval = portage.process.spawn([postsync, dosyncuri],
+ env=self.settings.environ())
+ if retval != os.EX_OK:
+ writemsg_level(" %s spawn failed of %s\n" % (bad("*"),
+ postsync,), level=logging.ERROR, noiselevel=-1)
+ return retval
+ return os.EX_OK
+
+
+ def pre_sync(self, repo):
+ self.settings, self.trees, self.mtimedb = self.emerge_config
+ self.xterm_titles = "notitles" not in self.settings.features
+ msg = ">>> Synchronization of repository '%s' located in '%s'..." \
+ % (repo.name, repo.location)
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n")
+ try:
+ st = os.stat(repo.location)
+ except OSError:
+ st = None
+ if st is None:
+ print(">>> '%s' not found, creating it." % repo.location)
+ portage.util.ensure_dirs(repo.location, mode=0o755)
+ st = os.stat(repo.location)
+
+ self.usersync_uid = None
+ spawn_kwargs = {}
+ spawn_kwargs["env"] = self.settings.environ()
+ if 'usersync' in self.settings.features and \
+ portage.data.secpass >= 2 and \
+ (st.st_uid != os.getuid() and st.st_mode & 0o700 or \
+ st.st_gid != os.getgid() and st.st_mode & 0o070):
+ try:
+ homedir = pwd.getpwuid(st.st_uid).pw_dir
+ except KeyError:
+ pass
+ else:
+ # Drop privileges when syncing, in order to match
+ # existing uid/gid settings.
+ self.usersync_uid = st.st_uid
+ spawn_kwargs["uid"] = st.st_uid
+ spawn_kwargs["gid"] = st.st_gid
+ spawn_kwargs["groups"] = [st.st_gid]
+ spawn_kwargs["env"]["HOME"] = homedir
+ umask = 0o002
+ if not st.st_mode & 0o020:
+ umask = umask | 0o020
+ spawn_kwargs["umask"] = umask
+ self.spawn_kwargs = spawn_kwargs
+
+ if self.usersync_uid is not None:
+ # PORTAGE_TMPDIR is used below, so validate it and
+ # bail out if necessary.
+ rval = _check_temp_dir(self.settings)
+ if rval != os.EX_OK:
+ return rval
+
+ os.umask(0o022)
+ return os.EX_OK
+
+
+ def _sync_callback(self, exitcode, updatecache_flg):
+ if updatecache_flg and "metadata-transfer" not in self.settings.features:
+ updatecache_flg = False
+
+ if updatecache_flg and \
+ os.path.exists(os.path.join(self.repo.location, 'metadata', 'cache')):
+
+ # Only update cache for repo.location since that's
+ # the only one that's been synced here.
+ action_metadata(self.settings, self.portdb, self.emerge_config.opts,
+ porttrees=[self.repo.location])
+
diff --git a/pym/portage/sync/getaddrinfo_validate.py b/pym/portage/sync/getaddrinfo_validate.py
new file mode 100644
index 0000000..5e6009c
--- /dev/null
+++ b/pym/portage/sync/getaddrinfo_validate.py
@@ -0,0 +1,29 @@
+# Copyright 2010 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import sys
+
+if sys.hexversion >= 0x3000000:
+ basestring = str
+
+def getaddrinfo_validate(addrinfos):
+ """
+ Validate structures returned from getaddrinfo(),
+ since they may be corrupt, especially when python
+ has IPv6 support disabled (bug #340899).
+ """
+ valid_addrinfos = []
+ for addrinfo in addrinfos:
+ try:
+ if len(addrinfo) != 5:
+ continue
+ if len(addrinfo[4]) < 2:
+ continue
+ if not isinstance(addrinfo[4][0], basestring):
+ continue
+ except TypeError:
+ continue
+
+ valid_addrinfos.append(addrinfo)
+
+ return valid_addrinfos
diff --git a/pym/portage/sync/modules/__init__.py b/pym/portage/sync/modules/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pym/portage/sync/modules/cvs/__init__.py b/pym/portage/sync/modules/cvs/__init__.py
new file mode 100644
index 0000000..fd3b12a
--- /dev/null
+++ b/pym/portage/sync/modules/cvs/__init__.py
@@ -0,0 +1,46 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+"""CVS plug-in module for portage.
+Performs a cvs up on repositories
+"""
+
+
+from portage.localization import _
+from portage.sync.config_checks import CheckSyncConfig
+from portage.util import writemsg_level
+
+
+class CheckCVSConfig(CheckSyncConfig):
+
+ def __init__(self, logger):
+ CheckSyncConfig.__init__(self, logger)
+ self.checks.append('check_cvs_repo')
+
+
+ def check_cvs_repo(self):
+ if self.repo.sync_cvs_repo is None:
+ writemsg_level("!!! %s\n" %
+ _("Repository '%s' has sync-type=cvs, but is missing sync-cvs-repo attribute")
+ % self.repo.name, level=self.logger.ERROR, noiselevel=-1)
+
+
+module_spec = {
+ 'name': 'cvs',
+ 'description': __doc__,
+ 'provides':{
+ 'cvs-module': {
+ 'name': "cvs",
+ 'class': "CVSSync",
+ 'description': __doc__,
+ 'functions': ['sync', 'new', 'exists'],
+ 'func_desc': {
+ 'sync': 'Performs a cvs up on the repository',
+ 'new': 'Creates the new repository at the specified location',
+ 'exists': 'Returns a boolean of whether the specified dir ' +
+ 'exists and is a valid CVS repository',
+ },
+ 'validate_config': CheckCVSConfig,
+ }
+ }
+}
diff --git a/pym/portage/sync/modules/cvs/cvs.py b/pym/portage/sync/modules/cvs/cvs.py
new file mode 100644
index 0000000..52d5d9a
--- /dev/null
+++ b/pym/portage/sync/modules/cvs/cvs.py
@@ -0,0 +1,82 @@
+# Copyright 1999-2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import errno
+
+import portage
+from portage import os
+from portage.util import writemsg_level
+from portage.sync.syncbase import SyncBase
+
+
+class CVSSync(SyncBase):
+ '''CVS sync module'''
+
+ short_desc = "Perform sync operations on CVS repositories"
+
+ @staticmethod
+ def name():
+ return "CVSSync"
+
+
+ def __init__(self):
+ SyncBase.__init__(self, "cvs", portage.const.CVS_PACKAGE_ATOM)
+
+
+ def new(self, **kwargs):
+ if kwargs:
+ self._kwargs(kwargs)
+ #initial checkout
+ msg = ">>> Starting initial cvs checkout with %s..." % self.repo.sync_uri
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n")
+ try:
+ os.rmdir(self.repo.location)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ msg = "!!! existing '%s' directory; exiting." % self.repo.location
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
+ return (1, False)
+ del e
+ cvs_root = self.repo.sync_uri
+ if portage.process.spawn_bash(
+ "cd %s; exec cvs -z0 -d %s co -P -d %s %s" %
+ (portage._shell_quote(os.path.dirname(self.repo.location)), portage._shell_quote(cvs_root),
+ portage._shell_quote(os.path.basename(self.repo.location)),
+ portage._shell_quote(self.repo.sync_cvs_repo)),
+ **portage._native_kwargs(self.spawn_kwargs)) != os.EX_OK:
+ msg = "!!! cvs checkout error; exiting."
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
+ return (1, False)
+ return (0, False)
+
+
+ def _sync(self):
+ """
+ Internal function to sync an existing CVS repository
+
+ @return: tuple of return code (0=success), whether the cache
+ needs to be updated
+ @rtype: (int, bool)
+ """
+
+ cvs_root = self.repo.sync_uri
+
+ if cvs_root.startswith("cvs://"):
+ cvs_root = cvs_root[6:]
+ #cvs update
+ msg = ">>> Starting cvs update with %s..." % self.repo.sync_uri
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n")
+ exitcode = portage.process.spawn_bash(
+ "cd %s; exec cvs -z0 -q update -dP" % \
+ (portage._shell_quote(self.repo.location),),
+ **portage._native_kwargs(self.spawn_kwargs))
+ if exitcode != os.EX_OK:
+ msg = "!!! cvs update error; exiting."
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
+ return (exitcode, False)
diff --git a/pym/portage/sync/modules/git/__init__.py b/pym/portage/sync/modules/git/__init__.py
new file mode 100644
index 0000000..304142e
--- /dev/null
+++ b/pym/portage/sync/modules/git/__init__.py
@@ -0,0 +1,30 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+"""Git plug-in module for portage.
+Performs a git pull on repositories
+"""
+
+
+from portage.sync.config_checks import CheckSyncConfig
+
+
+module_spec = {
+ 'name': 'git',
+ 'description': __doc__,
+ 'provides':{
+ 'git-module': {
+ 'name': "git",
+ 'class': "GitSync",
+ 'description': __doc__,
+ 'functions': ['sync', 'new', 'exists'],
+ 'func_desc': {
+ 'sync': 'Performs a git pull on the repository',
+ 'new': 'Creates the new repository at the specified location',
+ 'exists': 'Returns a boolean of whether the specified dir ' +
+ 'exists and is a valid Git repository',
+ },
+ 'validate_config': CheckSyncConfig,
+ }
+ }
+}
diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py
new file mode 100644
index 0000000..5e0e5df
--- /dev/null
+++ b/pym/portage/sync/modules/git/git.py
@@ -0,0 +1,124 @@
+# Copyright 2005-2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+
+import portage
+from portage import os
+from portage.util import writemsg_level
+from portage.output import create_color_func
+good = create_color_func("GOOD")
+bad = create_color_func("BAD")
+warn = create_color_func("WARN")
+from .timestamps import git_sync_timestamps
+from portage.sync.syncbase import SyncBase
+
+
+class GitSync(SyncBase):
+ '''Git sync class'''
+
+ short_desc = "Perform sync operations on git based repositories"
+
+ @staticmethod
+ def name():
+ return "GitSync"
+
+
+ def __init__(self):
+ SyncBase.__init__(self, "git", portage.const.GIT_PACKAGE_ATOM)
+
+
+ def exists(self, **kwargs):
+ '''Tests whether the repo actually exists'''
+ if kwargs:
+ self._kwargs(kwargs)
+ elif not self.repo:
+ return False
+
+ if not os.path.exists(self.repo.location):
+ return False
+ exitcode = portage.process.spawn_bash("cd %s ; git rev-parse" %\
+ (portage._shell_quote(self.repo.location),),
+ **portage._native_kwargs(self.spawn_kwargs))
+ if exitcode == 128:
+ return False
+ return True
+
+
+ def new(self, **kwargs):
+ '''Do the initial clone of the repository'''
+ if kwargs:
+ self._kwargs(kwargs)
+ emerge_config = self.options.get('emerge_config', None)
+ portdb = self.options.get('portdb', None)
+ try:
+ if not os.path.exists(self.repo.location):
+ os.makedirs(self.repo.location)
+ self.logger(self.xterm_titles,
+ 'Created new directory %s' % self.repo.location)
+ except IOError:
+ return (1, False)
+ msg = ">>> Cloning git repository from upstream into %s..." % self.repo.location
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n")
+ exitcode = portage.process.spawn_bash("cd %s ; %s clone %s ." % \
+ (portage._shell_quote(self.repo.location),
+ self.bin_command,
+ portage._shell_quote(self.repo.sync_uri)),
+ **portage._native_kwargs(self.spawn_kwargs))
+ if exitcode != os.EX_OK:
+ msg = "!!! git clone error in %s" % self.repo.location
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
+ return (exitcode, False)
+ msg = ">>> Git clone successful"
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n")
+ return self.post_sync(portdb, self.repo.location, emerge_config)
+
+
+ def _sync(self):
+ ''' Update existing git repository, and ignore the syncuri. We are
+ going to trust the user and assume that the user is in the branch
+ that he/she wants updated. We'll let the user manage branches with
+ git directly.
+ '''
+ # No kwargs call here; this is internal, so it should have been
+ # called by something which set the internal variables
+ emerge_config = self.options.get('emerge_config', None)
+ portdb = self.options.get('portdb', None)
+
+ msg = ">>> Starting git pull in %s..." % self.repo.location
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n")
+ exitcode = portage.process.spawn_bash("cd %s ; git pull" % \
+ (portage._shell_quote(self.repo.location),),
+ **portage._native_kwargs(self.spawn_kwargs))
+ if exitcode != os.EX_OK:
+ msg = "!!! git pull error in %s" % self.repo.location
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
+ return (exitcode, False)
+ msg = ">>> Git pull successful: %s" % self.repo.location
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n")
+ return self.post_sync(portdb, self.repo.location, emerge_config)
+
+
+ def post_sync(self, portdb, location, emerge_config):
+ '''repo.sync_type == "git":
+ # NOTE: Do this after reloading the config, in case
+ # it did not exist prior to sync, so that the config
+ # and portdb properly account for its existence.
+ '''
+ # avoid circular import for now
+ from _emerge.actions import load_emerge_config, adjust_configs
+ # Reload the whole config from scratch.
+ settings, trees, mtimedb = load_emerge_config(emerge_config=emerge_config)
+ adjust_configs(emerge_config.opts, emerge_config.trees)
+ portdb = trees[settings['EROOT']]['porttree'].dbapi
+ updatecache_flg = False
+ exitcode = git_sync_timestamps(portdb, location)
+ if exitcode == os.EX_OK:
+ updatecache_flg = True
+ return (exitcode, updatecache_flg)
diff --git a/pym/portage/sync/modules/git/timestamps.py b/pym/portage/sync/modules/git/timestamps.py
new file mode 100644
index 0000000..5e44845
--- /dev/null
+++ b/pym/portage/sync/modules/git/timestamps.py
@@ -0,0 +1,154 @@
+# Copyright 1999-2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import stat
+import subprocess
+
+
+import portage
+from portage import os
+from portage import _unicode_decode
+from portage.util import writemsg_level
+from portage.cache.cache_errors import CacheError
+
+
+def git_sync_timestamps(portdb, portdir):
+ """
+ Since git doesn't preserve timestamps, synchronize timestamps between
+ entries and ebuilds/eclasses. Assume the cache has the correct timestamp
+ for a given file as long as the file in the working tree is not modified
+ (relative to HEAD).
+ """
+
+ cache_db = portdb._pregen_auxdb.get(portdir)
+
+ try:
+ if cache_db is None:
+ # portdbapi does not populate _pregen_auxdb
+ # when FEATURES=metadata-transfer is enabled
+ cache_db = portdb._create_pregen_cache(portdir)
+ except CacheError as e:
+ writemsg_level("!!! Unable to instantiate cache: %s\n" % (e,),
+ level=logging.ERROR, noiselevel=-1)
+ return 1
+
+ if cache_db is None:
+ return os.EX_OK
+
+ if cache_db.validation_chf != 'mtime':
+ # newer formats like md5-dict do not require mtime sync
+ return os.EX_OK
+
+ writemsg_level(">>> Synchronizing timestamps...\n")
+
+ ec_dir = os.path.join(portdir, "eclass")
+ try:
+ ec_names = set(f[:-7] for f in os.listdir(ec_dir) \
+ if f.endswith(".eclass"))
+ except OSError as e:
+ writemsg_level("!!! Unable to list eclasses: %s\n" % (e,),
+ level=logging.ERROR, noiselevel=-1)
+ return 1
+
+ args = [portage.const.BASH_BINARY, "-c",
+ "cd %s && git diff-index --name-only --diff-filter=M HEAD" % \
+ portage._shell_quote(portdir)]
+ proc = subprocess.Popen(args, stdout=subprocess.PIPE)
+ modified_files = set(_unicode_decode(l).rstrip("\n") for l in proc.stdout)
+ rval = proc.wait()
+ proc.stdout.close()
+ if rval != os.EX_OK:
+ return rval
+
+ modified_eclasses = set(ec for ec in ec_names \
+ if os.path.join("eclass", ec + ".eclass") in modified_files)
+
+ updated_ec_mtimes = {}
+
+ for cpv in cache_db:
+ cpv_split = portage.catpkgsplit(cpv)
+ if cpv_split is None:
+ writemsg_level("!!! Invalid cache entry: %s\n" % (cpv,),
+ level=logging.ERROR, noiselevel=-1)
+ continue
+
+ cat, pn, ver, rev = cpv_split
+ cat, pf = portage.catsplit(cpv)
+ relative_eb_path = os.path.join(cat, pn, pf + ".ebuild")
+ if relative_eb_path in modified_files:
+ continue
+
+ try:
+ cache_entry = cache_db[cpv]
+ eb_mtime = cache_entry.get("_mtime_")
+ ec_mtimes = cache_entry.get("_eclasses_")
+ except KeyError:
+ writemsg_level("!!! Missing cache entry: %s\n" % (cpv,),
+ level=logging.ERROR, noiselevel=-1)
+ continue
+ except CacheError as e:
+ writemsg_level("!!! Unable to access cache entry: %s %s\n" % \
+ (cpv, e), level=logging.ERROR, noiselevel=-1)
+ continue
+
+ if eb_mtime is None:
+ writemsg_level("!!! Missing ebuild mtime: %s\n" % (cpv,),
+ level=logging.ERROR, noiselevel=-1)
+ continue
+
+ try:
+ eb_mtime = long(eb_mtime)
+ except ValueError:
+ writemsg_level("!!! Invalid ebuild mtime: %s %s\n" % \
+ (cpv, eb_mtime), level=logging.ERROR, noiselevel=-1)
+ continue
+
+ if ec_mtimes is None:
+ writemsg_level("!!! Missing eclass mtimes: %s\n" % (cpv,),
+ level=logging.ERROR, noiselevel=-1)
+ continue
+
+ if modified_eclasses.intersection(ec_mtimes):
+ continue
+
+ missing_eclasses = set(ec_mtimes).difference(ec_names)
+ if missing_eclasses:
+ writemsg_level("!!! Non-existent eclass(es): %s %s\n" % \
+ (cpv, sorted(missing_eclasses)), level=logging.ERROR,
+ noiselevel=-1)
+ continue
+
+ eb_path = os.path.join(portdir, relative_eb_path)
+ try:
+ current_eb_mtime = os.stat(eb_path)
+ except OSError:
+ writemsg_level("!!! Missing ebuild: %s\n" % \
+ (cpv,), level=logging.ERROR, noiselevel=-1)
+ continue
+
+ inconsistent = False
+ for ec, (ec_path, ec_mtime) in ec_mtimes.items():
+ updated_mtime = updated_ec_mtimes.get(ec)
+ if updated_mtime is not None and updated_mtime != ec_mtime:
+ writemsg_level("!!! Inconsistent eclass mtime: %s %s\n" % \
+ (cpv, ec), level=logging.ERROR, noiselevel=-1)
+ inconsistent = True
+ break
+
+ if inconsistent:
+ continue
+
+ if current_eb_mtime != eb_mtime:
+ os.utime(eb_path, (eb_mtime, eb_mtime))
+
+ for ec, (ec_path, ec_mtime) in ec_mtimes.items():
+ if ec in updated_ec_mtimes:
+ continue
+ ec_path = os.path.join(ec_dir, ec + ".eclass")
+ current_mtime = os.stat(ec_path)[stat.ST_MTIME]
+ if current_mtime != ec_mtime:
+ os.utime(ec_path, (ec_mtime, ec_mtime))
+ updated_ec_mtimes[ec] = ec_mtime
+
+ return os.EX_OK
diff --git a/pym/portage/sync/modules/rsync/__init__.py b/pym/portage/sync/modules/rsync/__init__.py
new file mode 100644
index 0000000..9e19d85
--- /dev/null
+++ b/pym/portage/sync/modules/rsync/__init__.py
@@ -0,0 +1,29 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+"""Rsync plug-in module for portage.
+ Performs rsync transfers on repositories
+"""
+
+
+from portage.sync.config_checks import CheckSyncConfig
+
+
+module_spec = {
+ 'name': 'rsync',
+ 'description': __doc__,
+ 'provides':{
+ 'rsync-module': {
+ 'name': "rsync",
+ 'class': "RsyncSync",
+ 'description': __doc__,
+ 'functions': ['sync', 'new', 'exists'],
+ 'func_desc': {
+ 'sync': 'Performs rsync transfers on the repository',
+ 'new': 'Creates the new repository at the specified location',
+ 'exists': 'Returns a boolean if the specified directory exists',
+ },
+ 'validate_config': CheckSyncConfig,
+ }
+ }
+ }
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
new file mode 100644
index 0000000..76d83f2
--- /dev/null
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -0,0 +1,519 @@
+# Copyright 1999-2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import sys
+import logging
+import time
+import signal
+import socket
+import re
+import random
+import tempfile
+
+import portage
+from portage import os
+from portage.util import writemsg_level
+from portage.output import create_color_func, yellow, blue, bold
+good = create_color_func("GOOD")
+bad = create_color_func("BAD")
+warn = create_color_func("WARN")
+from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
+from portage.util import writemsg, writemsg_stdout
+from portage.sync.getaddrinfo_validate import getaddrinfo_validate
+from _emerge.UserQuery import UserQuery
+from portage.sync.syncbase import SyncBase
+
+
+SERVER_OUT_OF_DATE = -1
+EXCEEDED_MAX_RETRIES = -2
+
+
+class RsyncSync(SyncBase):
+ '''Rsync sync module'''
+
+ short_desc = "Perform sync operations on rsync based repositories"
+
+ @staticmethod
+ def name():
+ return "RsyncSync"
+
+
+ def __init__(self):
+ SyncBase.__init__(self, "rsync", RSYNC_PACKAGE_ATOM)
+
+
+ def _sync(self):
+ '''Internal sync function which performs only the sync'''
+ opts = self.options.get('emerge_config').opts
+ self.usersync_uid = self.options.get('usersync_uid', None)
+ enter_invalid = '--ask-enter-invalid' in opts
+ out = portage.output.EOutput()
+ syncuri = self.repo.sync_uri
+ vcs_dirs = frozenset(VCS_DIRS)
+ vcs_dirs = vcs_dirs.intersection(os.listdir(self.repo.location))
+
+ for vcs_dir in vcs_dirs:
+ writemsg_level(("!!! %s appears to be under revision " + \
+ "control (contains %s).\n!!! Aborting rsync sync.\n") % \
+ (self.repo.location, vcs_dir), level=logging.ERROR, noiselevel=-1)
+ return (1, False)
+ self.timeout=180
+
+ rsync_opts = []
+ if self.settings["PORTAGE_RSYNC_OPTS"] == "":
+ rsync_opts = self._set_rsync_defaults()
+ else:
+ rsync_opts = self._validate_rsync_opts(rsync_opts, syncuri)
+ self.rsync_opts = self._rsync_opts_extend(opts, rsync_opts)
+
+ # Real local timestamp file.
+ self.servertimestampfile = os.path.join(
+ self.repo.location, "metadata", "timestamp.chk")
+
+ content = portage.util.grabfile(self.servertimestampfile)
+ timestamp = 0
+ if content:
+ try:
+ timestamp = time.mktime(time.strptime(content[0],
+ TIMESTAMP_FORMAT))
+ except (OverflowError, ValueError):
+ pass
+ del content
+
+ try:
+ self.rsync_initial_timeout = \
+ int(self.settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
+ except ValueError:
+ self.rsync_initial_timeout = 15
+
+ try:
+ maxretries=int(self.settings["PORTAGE_RSYNC_RETRIES"])
+ except SystemExit as e:
+ raise # Needed else can't exit
+ except:
+ maxretries = -1 #default number of retries
+
+ retries=0
+ try:
+ self.proto, user_name, hostname, port = re.split(
+ r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
+ syncuri, maxsplit=4)[1:5]
+ except ValueError:
+ writemsg_level("!!! sync-uri is invalid: %s\n" % syncuri,
+ noiselevel=-1, level=logging.ERROR)
+ return (1, False)
+
+ self.ssh_opts = self.settings.get("PORTAGE_SSH_OPTS")
+
+ if port is None:
+ port=""
+ if user_name is None:
+ user_name=""
+ if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
+ getaddrinfo_host = hostname
+ else:
+ # getaddrinfo needs the brackets stripped
+ getaddrinfo_host = hostname[1:-1]
+ updatecache_flg=True
+ all_rsync_opts = set(self.rsync_opts)
+ self.extra_rsync_opts = portage.util.shlex_split(
+ self.settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
+ all_rsync_opts.update(self.extra_rsync_opts)
+
+ family = socket.AF_UNSPEC
+ if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
+ family = socket.AF_INET
+ elif socket.has_ipv6 and \
+ ("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
+ family = socket.AF_INET6
+
+ addrinfos = None
+ uris = []
+
+ try:
+ addrinfos = getaddrinfo_validate(
+ socket.getaddrinfo(getaddrinfo_host, None,
+ family, socket.SOCK_STREAM))
+ except socket.error as e:
+ writemsg_level(
+ "!!! getaddrinfo failed for '%s': %s\n" % (hostname, e),
+ noiselevel=-1, level=logging.ERROR)
+
+ if addrinfos:
+
+ AF_INET = socket.AF_INET
+ AF_INET6 = None
+ if socket.has_ipv6:
+ AF_INET6 = socket.AF_INET6
+
+ ips_v4 = []
+ ips_v6 = []
+
+ for addrinfo in addrinfos:
+ if addrinfo[0] == AF_INET:
+ ips_v4.append("%s" % addrinfo[4][0])
+ elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
+ # IPv6 addresses need to be enclosed in square brackets
+ ips_v6.append("[%s]" % addrinfo[4][0])
+
+ random.shuffle(ips_v4)
+ random.shuffle(ips_v6)
+
+ # Give priority to the address family that
+ # getaddrinfo() returned first.
+ if AF_INET6 is not None and addrinfos and \
+ addrinfos[0][0] == AF_INET6:
+ ips = ips_v6 + ips_v4
+ else:
+ ips = ips_v4 + ips_v6
+
+ for ip in ips:
+ uris.append(syncuri.replace(
+ "//" + user_name + hostname + port + "/",
+ "//" + user_name + ip + port + "/", 1))
+
+ if not uris:
+ # With some configurations we need to use the plain hostname
+ # rather than try to resolve the ip addresses (bug #340817).
+ uris.append(syncuri)
+
+ # reverse, for use with pop()
+ uris.reverse()
+
+ effective_maxretries = maxretries
+ if effective_maxretries < 0:
+ effective_maxretries = len(uris) - 1
+
+ while (1):
+ if uris:
+ dosyncuri = uris.pop()
+ else:
+ writemsg("!!! Exhausted addresses for %s\n" % \
+ hostname, noiselevel=-1)
+ return (1, False)
+
+ if (retries==0):
+ if "--ask" in opts:
+ uq = UserQuery(opts)
+ if uq.query("Do you want to sync your Portage tree " + \
+ "with the mirror at\n" + blue(dosyncuri) + bold("?"),
+ enter_invalid) == "No":
+ print()
+ print("Quitting.")
+ print()
+ sys.exit(128 + signal.SIGINT)
+ self.logger(self.xterm_titles,
+ ">>> Starting rsync with " + dosyncuri)
+ if "--quiet" not in opts:
+ print(">>> Starting rsync with "+dosyncuri+"...")
+ else:
+ self.logger(self.xterm_titles,
+ ">>> Starting retry %d of %d with %s" % \
+ (retries, effective_maxretries, dosyncuri))
+ writemsg_stdout(
+ "\n\n>>> Starting retry %d of %d with %s\n" % \
+ (retries, effective_maxretries, dosyncuri), noiselevel=-1)
+
+ if dosyncuri.startswith('ssh://'):
+ dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
+
+ is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+ if is_synced:
+ break
+
+ retries=retries+1
+
+ if maxretries < 0 or retries <= maxretries:
+ print(">>> Retrying...")
+ else:
+ # over retries
+ # exit loop
+ updatecache_flg=False
+ exitcode = EXCEEDED_MAX_RETRIES
+ break
+ self._process_exitcode(exitcode, dosyncuri, out, maxretries)
+ return (exitcode, updatecache_flg)
+
+
+ def _process_exitcode(self, exitcode, syncuri, out, maxretries):
+ if (exitcode==0):
+ self.logger(self.xterm_titles, "=== Sync completed with %s" % syncuri)
+ elif exitcode == SERVER_OUT_OF_DATE:
+ exitcode = 1
+ elif exitcode == EXCEEDED_MAX_RETRIES:
+ sys.stderr.write(
+ ">>> Exceeded PORTAGE_RSYNC_RETRIES: %s\n" % maxretries)
+ exitcode = 1
+ elif (exitcode>0):
+ msg = []
+ if exitcode==1:
+ msg.append("Rsync has reported that there is a syntax error. Please ensure")
+ msg.append("that sync-uri attribute for repository '%s' is proper." % self.repo.name)
+ msg.append("sync-uri: '%s'" % self.repo.sync_uri)
+ elif exitcode==11:
+ msg.append("Rsync has reported that there is a File IO error. Normally")
+ msg.append("this means your disk is full, but can be caused by corruption")
+ msg.append("on the filesystem that contains repository '%s'. Please investigate" % self.repo.name)
+ msg.append("and try again after the problem has been fixed.")
+ msg.append("Location of repository: '%s'" % self.repo.location)
+ elif exitcode==20:
+ msg.append("Rsync was killed before it finished.")
+ else:
+ msg.append("Rsync has not successfully finished. It is recommended that you keep")
+ msg.append("trying or that you use the 'emerge-webrsync' option if you are unable")
+ msg.append("to use rsync due to firewall or other restrictions. This should be a")
+ msg.append("temporary problem unless complications exist with your network")
+ msg.append("(and possibly your system's filesystem) configuration.")
+ for line in msg:
+ out.eerror(line)
+
+
+ def new(self, **kwargs):
+ if kwargs:
+ self._kwargs(kwargs)
+ try:
+ if not os.path.exists(self.repo.location):
+ os.makedirs(self.repo.location)
+ self.logger(self.self.xterm_titles,
+ 'Created New Directory %s ' % self.repo.location )
+ except IOError:
+ return (1, False)
+ return self._sync()
+
+
+ def _set_rsync_defaults(self):
+ portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
+ rsync_opts = [
+ "--recursive", # Recurse directories
+ "--links", # Consider symlinks
+ "--safe-links", # Ignore links outside of tree
+ "--perms", # Preserve permissions
+ "--times", # Preserive mod times
+ "--omit-dir-times",
+ "--compress", # Compress the data transmitted
+ "--force", # Force deletion on non-empty dirs
+ "--whole-file", # Don't do block transfers, only entire files
+ "--delete", # Delete files that aren't in the master tree
+ "--stats", # Show final statistics about what was transfered
+ "--human-readable",
+ "--timeout="+str(self.timeout), # IO timeout if not done in X seconds
+ "--exclude=/distfiles", # Exclude distfiles from consideration
+ "--exclude=/local", # Exclude local from consideration
+ "--exclude=/packages", # Exclude packages from consideration
+ ]
+ return rsync_opts
+
+
+ def _validate_rsync_opts(self, rsync_opts, syncuri):
+ # The below validation is not needed when using the above hardcoded
+ # defaults.
+
+ portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
+ rsync_opts.extend(portage.util.shlex_split(
+ self.settings.get("PORTAGE_RSYNC_OPTS", "")))
+ for opt in ("--recursive", "--times"):
+ if opt not in rsync_opts:
+ portage.writemsg(yellow("WARNING:") + " adding required option " + \
+ "%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+ rsync_opts.append(opt)
+
+ for exclude in ("distfiles", "local", "packages"):
+ opt = "--exclude=/%s" % exclude
+ if opt not in rsync_opts:
+ portage.writemsg(yellow("WARNING:") + \
+ " adding required option %s not included in " % opt + \
+ "PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
+ rsync_opts.append(opt)
+
+ if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
+ def rsync_opt_startswith(opt_prefix):
+ for x in rsync_opts:
+ if x.startswith(opt_prefix):
+ return (1, False)
+ return (0, False)
+
+ if not rsync_opt_startswith("--timeout="):
+ rsync_opts.append("--timeout=%d" % self.timeout)
+
+ for opt in ("--compress", "--whole-file"):
+ if opt not in rsync_opts:
+ portage.writemsg(yellow("WARNING:") + " adding required option " + \
+ "%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
+ rsync_opts.append(opt)
+ return rsync_opts
+
+
+ @staticmethod
+ def _rsync_opts_extend(opts, rsync_opts):
+ if "--quiet" in opts:
+ rsync_opts.append("--quiet") # Shut up a lot
+ else:
+ rsync_opts.append("--verbose") # Print filelist
+
+ if "--verbose" in opts:
+ rsync_opts.append("--progress") # Progress meter for each file
+
+ if "--debug" in opts:
+ rsync_opts.append("--checksum") # Force checksum on all files
+ return rsync_opts
+
+
+ def _do_rsync(self, syncuri, timestamp, opts):
+ is_synced = False
+ if timestamp != 0 and "--quiet" not in opts:
+ print(">>> Checking server timestamp ...")
+
+ rsynccommand = [self.bin_command] + self.rsync_opts + self.extra_rsync_opts
+
+ if self.proto == 'ssh' and self.ssh_opts:
+ rsynccommand.append("--rsh=ssh " + self.ssh_opts)
+
+ if "--debug" in opts:
+ print(rsynccommand)
+
+ exitcode = os.EX_OK
+ servertimestamp = 0
+ # Even if there's no timestamp available locally, fetch the
+ # timestamp anyway as an initial probe to verify that the server is
+ # responsive. This protects us from hanging indefinitely on a
+ # connection attempt to an unresponsive server which rsync's
+ # --timeout option does not prevent.
+
+ #if True:
+ # Temporary file for remote server timestamp comparison.
+ # NOTE: If FEATURES=usersync is enabled then the tempfile
+ # needs to be in a directory that's readable by the usersync
+ # user. We assume that PORTAGE_TMPDIR will satisfy this
+ # requirement, since that's not necessarily true for the
+ # default directory used by the tempfile module.
+ if self.usersync_uid is not None:
+ tmpdir = self.settings['PORTAGE_TMPDIR']
+ else:
+ # use default dir from tempfile module
+ tmpdir = None
+ fd, tmpservertimestampfile = \
+ tempfile.mkstemp(dir=tmpdir)
+ os.close(fd)
+ if self.usersync_uid is not None:
+ portage.util.apply_permissions(tmpservertimestampfile,
+ uid=self.usersync_uid)
+ command = rsynccommand[:]
+ command.append(syncuri.rstrip("/") + \
+ "/metadata/timestamp.chk")
+ command.append(tmpservertimestampfile)
+ content = None
+ pids = []
+ try:
+ # Timeout here in case the server is unresponsive. The
+ # --timeout rsync option doesn't apply to the initial
+ # connection attempt.
+ try:
+ if self.rsync_initial_timeout:
+ portage.exception.AlarmSignal.register(
+ self.rsync_initial_timeout)
+
+ pids.extend(portage.process.spawn(
+ command, returnpid=True,
+ **portage._native_kwargs(self.spawn_kwargs)))
+ exitcode = os.waitpid(pids[0], 0)[1]
+ if self.usersync_uid is not None:
+ portage.util.apply_permissions(tmpservertimestampfile,
+ uid=os.getuid())
+ content = portage.grabfile(tmpservertimestampfile)
+ finally:
+ if self.rsync_initial_timeout:
+ portage.exception.AlarmSignal.unregister()
+ try:
+ os.unlink(tmpservertimestampfile)
+ except OSError:
+ pass
+ except portage.exception.AlarmSignal:
+ # timed out
+ print('timed out')
+ # With waitpid and WNOHANG, only check the
+ # first element of the tuple since the second
+ # element may vary (bug #337465).
+ if pids and os.waitpid(pids[0], os.WNOHANG)[0] == 0:
+ os.kill(pids[0], signal.SIGTERM)
+ os.waitpid(pids[0], 0)
+ # This is the same code rsync uses for timeout.
+ exitcode = 30
+ else:
+ if exitcode != os.EX_OK:
+ if exitcode & 0xff:
+ exitcode = (exitcode & 0xff) << 8
+ else:
+ exitcode = exitcode >> 8
+
+ if content:
+ try:
+ servertimestamp = time.mktime(time.strptime(
+ content[0], TIMESTAMP_FORMAT))
+ except (OverflowError, ValueError):
+ pass
+ del command, pids, content
+
+ if exitcode == os.EX_OK:
+ if (servertimestamp != 0) and (servertimestamp == timestamp):
+ self.logger(self.xterm_titles,
+ ">>> Cancelling sync -- Already current.")
+ print()
+ print(">>>")
+ print(">>> Timestamps on the server and in the local repository are the same.")
+ print(">>> Cancelling all further sync action. You are already up to date.")
+ print(">>>")
+ print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+ print(">>>")
+ print()
+ return is_synced, exitcode
+ elif (servertimestamp != 0) and (servertimestamp < timestamp):
+ self.logger(self.xterm_titles,
+ ">>> Server out of date: %s" % syncuri)
+ print()
+ print(">>>")
+ print(">>> SERVER OUT OF DATE: %s" % syncuri)
+ print(">>>")
+ print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
+ print(">>>")
+ print()
+ exitcode = SERVER_OUT_OF_DATE
+ elif (servertimestamp == 0) or (servertimestamp > timestamp):
+ # actual sync
+ command = rsynccommand + [syncuri+"/", self.repo.location]
+ exitcode = None
+ try:
+ exitcode = portage.process.spawn(command,
+ **portage._native_kwargs(self.spawn_kwargs))
+ finally:
+ if exitcode is None:
+ # interrupted
+ exitcode = 128 + signal.SIGINT
+
+ # 0 Success
+ # 1 Syntax or usage error
+ # 2 Protocol incompatibility
+ # 5 Error starting client-server protocol
+ # 35 Timeout waiting for daemon connection
+ if exitcode not in (0, 1, 2, 5, 35):
+ # If the exit code is not among those listed above,
+ # then we may have a partial/inconsistent sync
+ # state, so our previously read timestamp as well
+ # as the corresponding file can no longer be
+ # trusted.
+ timestamp = 0
+ try:
+ os.unlink(self.servertimestampfile)
+ except OSError:
+ pass
+
+ if exitcode in [0,1,3,4,11,14,20,21]:
+ is_synced = True
+ elif exitcode in [1,3,4,11,14,20,21]:
+ is_synced = True
+ else:
+ # Code 2 indicates protocol incompatibility, which is expected
+ # for servers with protocol < 29 that don't support
+ # --prune-empty-directories. Retry for a server that supports
+ # at least rsync protocol version 29 (>=rsync-2.6.4).
+ pass
+ return is_synced, exitcode
diff --git a/pym/portage/sync/modules/svn/__init__.py b/pym/portage/sync/modules/svn/__init__.py
new file mode 100644
index 0000000..1fda55a
--- /dev/null
+++ b/pym/portage/sync/modules/svn/__init__.py
@@ -0,0 +1,32 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+"""SVN plug-in module for portage.
+Performs a svn up on repositories
+"""
+
+
+from portage.localization import _
+from portage.sync.config_checks import CheckSyncConfig
+from portage.util import writemsg_level
+
+
+module_spec = {
+ 'name': 'svn',
+ 'description': __doc__,
+ 'provides':{
+ 'svn-module': {
+ 'name': "svn",
+ 'class': "SVNSync",
+ 'description': __doc__,
+ 'functions': ['sync', 'new', 'exists'],
+ 'func_desc': {
+ 'sync': 'Performs a svn up on the repository',
+ 'new': 'Creates the new repository at the specified location',
+ 'exists': 'Returns a boolean of whether the specified dir ' +
+ 'exists and is a valid SVN repository',
+ },
+ 'validate_config': CheckSyncConfig,
+ }
+ }
+}
diff --git a/pym/portage/sync/modules/svn/svn.py b/pym/portage/sync/modules/svn/svn.py
new file mode 100644
index 0000000..0365e90
--- /dev/null
+++ b/pym/portage/sync/modules/svn/svn.py
@@ -0,0 +1,104 @@
+# Copyright 1999-2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import logging
+import errno
+
+import portage
+from portage import os
+from portage.util import writemsg_level
+from portage.sync.syncbase import SyncBase
+
+
+class SVNSync(SyncBase):
+ '''SVN sync module'''
+
+ short_desc = "Perform sync operations on SVN repositories"
+
+ @staticmethod
+ def name():
+ return "SVNSync"
+
+
+ def __init__(self):
+ SyncBase.__init__(self, "svn", "dev-vcs/subversion")
+
+
+ def new(self, **kwargs):
+ if kwargs:
+ self._kwargs(kwargs)
+ #initial checkout
+ msg = ">>> Starting initial svn checkout with %s..." % self.repo.sync_uri
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n")
+ try:
+ os.rmdir(self.repo.location)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ msg = "!!! existing '%s' directory; exiting." % self.repo.location
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
+ return (1, False)
+ del e
+ svn_root = self.repo.sync_uri
+ exitcode = portage.process.spawn_bash(
+ "cd %s; exec svn %s" %
+ (portage._shell_quote(os.path.dirname(self.repo.location)),
+ portage._shell_quote(svn_root)),
+ **portage._native_kwargs(self.spawn_kwargs))
+ if exitcode != os.EX_OK:
+ msg = "!!! svn checkout error; exiting."
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
+ return (exitcode, False)
+
+
+ def _sync(self):
+ """
+ Internal function to sync an existing SVN repository
+
+ @return: tuple of return code (0=success), whether the cache
+ needs to be updated
+ @rtype: (int, bool)
+ """
+
+ exitcode, d = self._svn_upgrade()
+ if exitcode != os.EX_OK:
+ return (exitcode, False)
+
+ svn_root = self.repo.sync_uri
+
+ if svn_root.startswith("svn://"):
+ svn_root = svn_root[6:]
+ #svn update
+ msg = ">>> Starting svn update with %s..." % self.repo.sync_uri
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n")
+ exitcode = portage.process.spawn_bash(
+ "cd %s; exec svn update" % \
+ (portage._shell_quote(self.repo.location),),
+ **portage._native_kwargs(self.spawn_kwargs))
+ if exitcode != os.EX_OK:
+ msg = "!!! svn update error; exiting."
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
+ return (exitcode, False)
+
+
+ def _svn_upgrade(self):
+ """
+ Internal function which performs an svn upgrade on the repo
+
+ @return: tuple of return code (0=success), whether the cache
+ needs to be updated
+ @rtype: (int, bool)
+ """
+ exitcode = portage.process.spawn_bash(
+ "cd %s; exec svn upgrade" %
+ (portage._shell_quote(self.repo.location),),
+ **portage._native_kwargs(self.spawn_kwargs))
+ if exitcode != os.EX_OK:
+ msg = "!!! svn upgrade error; exiting."
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
+ return (exitcode, False)
diff --git a/pym/portage/sync/modules/websync/__init__.py b/pym/portage/sync/modules/websync/__init__.py
new file mode 100644
index 0000000..e93ee10
--- /dev/null
+++ b/pym/portage/sync/modules/websync/__init__.py
@@ -0,0 +1,48 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+"""WebRSync plug-in module for portage.
+Performs a http download of a portage snapshot, verifies and
+unpacks it to the repo location.
+"""
+
+import os
+
+from portage.sync.config_checks import CheckSyncConfig
+
+
+DEFAULT_CLASS = "WebRsync"
+AVAILABLE_CLASSES = [ "WebRsync", "PyWebsync"]
+options = {"1": "WebRsync", "2": "PyWebsync"}
+
+
+config_class = DEFAULT_CLASS
+try:
+ test_param = os.environ["TESTIT"]
+ if test_param in options:
+ config_class = options[test_param]
+except KeyError:
+ pass
+
+
+module_spec = {
+ 'name': 'webrsync',
+ 'description': __doc__,
+ 'provides':{
+ 'websync-module': {
+ 'name': "websync",
+ 'class': config_class,
+ 'description': __doc__,
+ 'functions': ['sync', 'new', 'exists'],
+ 'func_desc': {
+ 'sync': 'Performs an archived http download of the ' +
+ 'repository, then unpacks it. Optionally it performs a ' +
+ 'gpg verification of the downloaded file(s)',
+ 'new': 'Creates the new repository at the specified location',
+ 'exists': 'Returns a boolean of whether the specified dir ' +
+ 'exists and is a valid repository',
+ },
+ 'validate_config': CheckSyncConfig,
+ },
+ }
+}
diff --git a/pym/portage/sync/modules/websync/websync.py b/pym/portage/sync/modules/websync/websync.py
new file mode 100644
index 0000000..f08ae77
--- /dev/null
+++ b/pym/portage/sync/modules/websync/websync.py
@@ -0,0 +1,81 @@
+
+'''WebRsync module for portage'''
+
+import logging
+
+import portage
+from portage import os
+from portage.util import writemsg_level
+from portage.output import create_color_func
+good = create_color_func("GOOD")
+bad = create_color_func("BAD")
+warn = create_color_func("WARN")
+from portage.sync.syncbase import SyncBase
+
+
+class WebRsync(SyncBase):
+ '''WebRSync sync class'''
+
+ short_desc = "Perform sync operations on webrsync based repositories"
+
+ @staticmethod
+ def name():
+ return "WebRSync"
+
+
+ def __init__(self):
+ SyncBase.__init__(self, 'emerge-webrsync', '>=sys-apps/portage-2.3')
+
+
+ def new(self, **kwargs):
+ '''Do the initial download and install of the repository'''
+ return self._sync()
+
+
+ def _sync(self):
+ ''' Update existing repository
+ '''
+ emerge_config = self.options.get('emerge_config', None)
+ portdb = self.options.get('portdb', None)
+
+ msg = ">>> Starting emerge-webrsync for %s..." % self.repo.location
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n")
+ exitcode = portage.process.spawn_bash("%s" % \
+ (self.bin_command),
+ **portage._native_kwargs(self.spawn_kwargs))
+ if exitcode != os.EX_OK:
+ msg = "!!! emerge-webrsync error in %s" % self.repo.location
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
+ return (exitcode, False)
+ msg = ">>> Emerge-webrsync successful: %s" % self.repo.location
+ self.logger(self.xterm_titles, msg)
+ writemsg_level(msg + "\n")
+ #return self.post_sync(portdb, self.repo.location, emerge_config)
+ return (exitcode, True)
+
+
+class PyWebRsync(SyncBase):
+ '''WebRSync sync class'''
+
+ short_desc = "Perform sync operations on webrsync based repositories"
+
+ @staticmethod
+ def name():
+ return "WebRSync"
+
+
+ def __init__(self):
+ SyncBase.__init__(self, None, '>=sys-apps/portage-2.3')
+
+
+ def new(self, **kwargs):
+ '''Do the initial download and install of the repository'''
+ pass
+
+
+ def _sync(self):
+ ''' Update existing repository
+ '''
+ pass
diff --git a/pym/portage/sync/old_tree_timestamp.py b/pym/portage/sync/old_tree_timestamp.py
new file mode 100644
index 0000000..aa23a27
--- /dev/null
+++ b/pym/portage/sync/old_tree_timestamp.py
@@ -0,0 +1,100 @@
+# Copyright 2010-2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import division
+
+import locale
+import logging
+import time
+
+from portage import os
+from portage.exception import PortageException
+from portage.localization import _
+from portage.output import EOutput
+from portage.util import grabfile, writemsg_level
+
+def have_english_locale():
+ lang, enc = locale.getdefaultlocale()
+ if lang is not None:
+ lang = lang.lower()
+ lang = lang.split('_', 1)[0]
+ return lang is None or lang in ('c', 'en')
+
+def whenago(seconds):
+ sec = int(seconds)
+ mins = 0
+ days = 0
+ hrs = 0
+ years = 0
+ out = []
+
+ if sec > 60:
+ mins = sec // 60
+ sec = sec % 60
+ if mins > 60:
+ hrs = mins // 60
+ mins = mins % 60
+ if hrs > 24:
+ days = hrs // 24
+ hrs = hrs % 24
+ if days > 365:
+ years = days // 365
+ days = days % 365
+
+ if years:
+ out.append("%dy " % years)
+ if days:
+ out.append("%dd " % days)
+ if hrs:
+ out.append("%dh " % hrs)
+ if mins:
+ out.append("%dm " % mins)
+ if sec:
+ out.append("%ds " % sec)
+
+ return "".join(out).strip()
+
+def old_tree_timestamp_warn(portdir, settings):
+ unixtime = time.time()
+ default_warnsync = 30
+
+ timestamp_file = os.path.join(portdir, "metadata/timestamp.x")
+ try:
+ lastsync = grabfile(timestamp_file)
+ except PortageException:
+ return False
+
+ if not lastsync:
+ return False
+
+ lastsync = lastsync[0].split()
+ if not lastsync:
+ return False
+
+ try:
+ lastsync = int(lastsync[0])
+ except ValueError:
+ return False
+
+ var_name = 'PORTAGE_SYNC_STALE'
+ try:
+ warnsync = float(settings.get(var_name, default_warnsync))
+ except ValueError:
+ writemsg_level("!!! %s contains non-numeric value: %s\n" % \
+ (var_name, settings[var_name]),
+ level=logging.ERROR, noiselevel=-1)
+ return False
+
+ if warnsync <= 0:
+ return False
+
+ if (unixtime - 86400 * warnsync) > lastsync:
+ out = EOutput()
+ if have_english_locale():
+ out.ewarn("Last emerge --sync was %s ago." % \
+ whenago(unixtime - lastsync))
+ else:
+ out.ewarn(_("Last emerge --sync was %s.") % \
+ time.strftime('%c', time.localtime(lastsync)))
+ return True
+ return False
diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
new file mode 100644
index 0000000..94d4aab
--- /dev/null
+++ b/pym/portage/sync/syncbase.py
@@ -0,0 +1,102 @@
+# Copyright 2014 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+'''
+Base class for performing sync operations.
+This class contains common initialization code and functions.
+'''
+
+
+import os
+
+import portage
+from portage.util import writemsg_level
+
+
+class SyncBase(object):
+ '''Base Sync class for subclassing'''
+
+ short_desc = "Perform sync operations on repositories"
+
+ @staticmethod
+ def name():
+ return "BlankSync"
+
+
+ def can_progressbar(self, func):
+ return False
+
+
+ def __init__(self, bin_command, bin_pkg):
+ self.options = None
+ self.settings = None
+ self.logger = None
+ self.repo = None
+ self.xterm_titles = None
+ self.spawn_kwargs = None
+ self.bin_command = None
+ self.has_bin = False
+ if bin_command:
+ self.bin_command = portage.process.find_binary(bin_command)
+ if self.bin_command is None:
+ msg = ["Command not found: %s" % bin_command,
+ "Type \"emerge %s\" to enable %s support." % (bin_pkg, bin_command)]
+ for l in msg:
+ writemsg_level("!!! %s\n" % l,
+ level=self.logger.ERROR, noiselevel=-1)
+ else:
+ self.has_bin = True
+
+
+ def _kwargs(self, kwargs):
+ '''Sets internal variables from kwargs'''
+ self.options = kwargs.get('options', {})
+ self.settings = self.options.get('settings', None)
+ self.logger = self.options.get('logger', None)
+ self.repo = self.options.get('repo', None)
+ self.xterm_titles = self.options.get('xterm_titles', False)
+ self.spawn_kwargs = self.options.get('spawn_kwargs', None)
+
+
+ def exists(self, **kwargs):
+ '''Tests whether the repo actually exists'''
+ if kwargs:
+ self._kwargs(kwargs)
+ elif not self.repo:
+ return False
+
+
+ if not os.path.exists(self.repo.location):
+ return False
+ return True
+
+
+ def sync(self, **kwargs):
+ '''Sync the repository'''
+ if kwargs:
+ self._kwargs(kwargs)
+
+ if not self.has_bin:
+ return (1, False)
+
+ if not self.exists():
+ return self.new()
+ return self._sync()
+
+
+ def new(self, **kwargs):
+ '''Do the initial download and install of the repository'''
+ pass
+
+ def _sync(self):
+ '''Update existing repository
+ '''
+ pass
+
+ def post_sync(self, portdb, location, emerge_config):
+ '''repo.sync_type == "Blank":
+ # NOTE: Do this after reloading the config, in case
+ # it did not exist prior to sync, so that the config
+ # and portdb properly account for its existence.
+ '''
+ pass
