This patch makes the a-a-analyze-vmcore to use the suided helper abrt-action-debuginfo-install-to-abrt-cache to install vmcore debuginfo to /var/cache/abrt-di. Please note that the minor-fix.path is meant for libreport. Run-tested, please review.
Thank you, Jirka
>From f54419e342c8b9c80a0b84133f0458cb8271d9fd Mon Sep 17 00:00:00 2001 From: Jiri Moskovcak <[email protected]> Date: Mon, 20 Feb 2012 16:18:37 +0100 Subject: [PATCH] debuginfo downloader - minor fix --- src/client-python/debuginfo.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client-python/debuginfo.py b/src/client-python/debuginfo.py index 634fde6..44e3d8b 100644 --- a/src/client-python/debuginfo.py +++ b/src/client-python/debuginfo.py @@ -156,7 +156,7 @@ class DebugInfoDownload(YumBase): unmute_stdout() # return value will be used as exitcode. So 0 = ok, !0 - error - def download(self, files, exact_files=False): + def download(self, files, download_exact_files=False): """ @files - """ installed_size = 0 total_pkgs = 0 @@ -291,7 +291,7 @@ class DebugInfoDownload(YumBase): print (_("Downloading package {0} failed").format(pkg)) else: unpack_result = unpack_rpm(package_file_name, files, self.tmpdir, - self.cachedir, self.keeprpms, exact_files=exact_files) + self.cachedir, self.keeprpms, exact_files=download_exact_files) if unpack_result == RETURN_FAILURE: # recursively delete the temp dir on failure print _("Unpacking failed, aborting download...") -- 1.7.7.6
>From 7ef835a988307064b91c8e8fca20ac619f0208a1 Mon Sep 17 00:00:00 2001 From: Jiri Moskovcak <[email protected]> Date: Mon, 20 Feb 2012 16:21:07 +0100 Subject: [PATCH] use the suided helper for installing vmcore debuginfo --- src/plugins/abrt-action-analyze-vmcore.in | 22 +- .../abrt-action-install-debuginfo-to-abrt-cache.c | 3 + src/plugins/abrt-action-install-debuginfo.in | 455 +++----------------- 3 files changed, 69 insertions(+), 411 deletions(-) diff --git a/src/plugins/abrt-action-analyze-vmcore.in b/src/plugins/abrt-action-analyze-vmcore.in index 7c96454..8441cc8 100644 --- a/src/plugins/abrt-action-analyze-vmcore.in +++ b/src/plugins/abrt-action-analyze-vmcore.in @@ -7,8 +7,9 @@ from subprocess import Popen, PIPE import os import sys from reportclient import _, verbose, log, log1, log2, set_verbosity, error_msg_and_die -from reportclient.debuginfo import DebugInfoDownload import time +from reportclient.debuginfo import DebugInfoDownload +import getopt PROGNAME = "abrt-action-analyze-vmcore" @@ -33,6 +34,7 @@ if __name__ == "__main__": except getopt.GetoptError, err: error_msg(err) # prints something like "option -a not recognized" error_msg_and_die(help_text) + usercache = False for opt, arg in opts: if opt in ("-h", "--help"): @@ -40,6 +42,7 @@ if __name__ == "__main__": exit(0) if opt == "--cache": cachedirs = arg.split(':') + usercache = True elif opt == "-v": verbose += 1 elif opt == "--core": @@ -96,11 +99,18 @@ if __name__ == "__main__": print _("Installing kernel debuginfo(s)") # replace with abrt-action-install-debuginfo vmlinux_di_path = "/usr/lib/debug/lib/modules/%s/vmlinux" % ver - downloader = DebugInfoDownload(cache=vmlinux_di_cachedir, tmp=tmpdir, keep_rpms=True) - res = downloader.download([vmlinux_di_path], exact_files=True) - if res != RETURN_OK: - error_msg_and_die(_("Can't download required debuginfo")) - vmlinux_di_path = "{0}/usr/lib/debug/lib/modules/{1}/vmlinux".format(cachedir, ver) + if usercache: # if user set the cache, we assume it's writeable for him + downloader = DebugInfoDownload(cache=vmlinux_di_cachedir, tmp=tmpdir, keep_rpms=True) + res = downloader.download([vmlinux_di_path], download_exact_files=True) + if res != RETURN_OK: + error_msg_and_die(_("Can't download required debuginfo")) + vmlinux_di_path = "{0}/usr/lib/debug/lib/modules/{1}/vmlinux".format(cachedir, ver) + else: # we need to use the suid wrapper, if we use the default cache + downloader = Popen(["/usr/libexec/abrt-action-install-debuginfo-to-abrt-cache", "--size_mb=4096", "--exact={0}".format(vmlinux_di_path)], bufsize = -1) + res = downloader.wait() + if res != 0: + error_msg_and_die(_("Can't download required debuginfo")) + vmlinux_di_path = "{0}/usr/lib/debug/lib/modules/{1}/vmlinux".format(cachedir, ver) print _("Generating backtrace") log_file = open("kernel_log","w") diff --git a/src/plugins/abrt-action-install-debuginfo-to-abrt-cache.c b/src/plugins/abrt-action-install-debuginfo-to-abrt-cache.c index 96655c2..080de27 100644 --- a/src/plugins/abrt-action-install-debuginfo-to-abrt-cache.c +++ b/src/plugins/abrt-action-install-debuginfo-to-abrt-cache.c @@ -41,6 +41,9 @@ int main(int argc, char **argv) if (strcmp(arg, "--ids=-") == 0) continue; + if (strncmp(arg, "--exact", 7) == 0) + continue; + if (strncmp(arg, "--cache", 7) == 0) error_msg_and_die("bad option %s", arg); if (strncmp(arg, "--tmpdir", 8) == 0) diff --git a/src/plugins/abrt-action-install-debuginfo.in b/src/plugins/abrt-action-install-debuginfo.in index 0da5117..280d815 100644 --- a/src/plugins/abrt-action-install-debuginfo.in +++ b/src/plugins/abrt-action-install-debuginfo.in @@ -8,13 +8,12 @@ PROGNAME = "abrt-action-install-debuginfo" import sys import os import errno -import time import getopt -import shutil import reportclient from subprocess import Popen, PIPE -from yum import _, YumBase -from yum.callbacks import DownloadBaseCallback +from reportclient import _, verbose, log, log1, log2, set_verbosity, error_msg_and_die, error_msg +import time +from reportclient.debuginfo import DebugInfoDownload, filter_installed_debuginfos, build_ids_to_path # everything was ok RETURN_OK = 0 @@ -42,371 +41,6 @@ def init_gettext(): gettext.bindtextdomain(GETTEXT_PROGNAME, '/usr/share/locale') gettext.textdomain(GETTEXT_PROGNAME) - -verbose = 0 -def log(fmt, *args): - sys.stderr.write("%s\n" % (fmt % args)) - -def log1(fmt, *args): - """ prints log message if verbosity >= 1 """ - if verbose >= 1: - sys.stderr.write("%s\n" % (fmt % args)) - -def log2(fmt, *args): - """ prints log message if verbosity >= 2 """ - if verbose >= 2: - sys.stderr.write("%s\n" % (fmt % args)) - -def error_msg(fmt, *args): - sys.stderr.write("%s\n" % (fmt % args)) - -def error_msg_and_die(fmt, *args): - sys.stderr.write("%s\n" % (fmt % args)) - sys.exit(1) - - -old_stdout = -1 -def mute_stdout(): - if verbose < 2: - global old_stdout - old_stdout = sys.stdout - sys.stdout = open("/dev/null", "w") - -def unmute_stdout(): - if verbose < 2: - if old_stdout != -1: - sys.stdout = old_stdout - else: - print "ERR: unmute called without mute?" - -# TODO: unpack just required debuginfo and not entire rpm? -# ..that can lead to: foo.c No such file and directory -# files is not used... -def unpack_rpm(package_file_name, files, tmp_dir, destdir, keeprpm): - package_full_path = tmp_dir + "/" + package_file_name - log1("Extracting %s to %s", package_full_path, destdir) - log2("%s", files) - print _("Extracting cpio from {0}").format(package_full_path) - unpacked_cpio_path = tmp_dir + "/unpacked.cpio" - try: - unpacked_cpio = open(unpacked_cpio_path, 'wb') - except IOError, ex: - print _("Can't write to '{0}': {1}").format(unpacked_cpio_path, ex) - return RETURN_FAILURE - rpm2cpio = Popen(["rpm2cpio", package_full_path], - stdout = unpacked_cpio, bufsize = -1) - retcode = rpm2cpio.wait() - - if retcode == 0: - log1("cpio written OK") - if not keeprpm: - log1("keeprpms = False, removing %s", package_full_path) - #print _("Removing temporary rpm file") - os.unlink(package_full_path) - else: - unpacked_cpio.close() - print _("Can't extract package '{0}'").format(package_full_path) - return RETURN_FAILURE - - # close the file - unpacked_cpio.close() - # and open it for reading - unpacked_cpio = open(unpacked_cpio_path, 'rb') - - print _("Caching files from {0} made from {1}").format("unpacked.cpio", package_file_name) - cpio = Popen(["cpio", "-idu", "--quiet"], - stdin=unpacked_cpio, cwd=destdir, bufsize=-1) - retcode = cpio.wait() - - if retcode == 0: - log1("files extracted OK") - #print _("Removing temporary cpio file") - os.unlink(unpacked_cpio_path) - else: - print _("Can't extract files from '{0}'").format(unpacked_cpio_path) - return RETURN_FAILURE - -class MyDownloadCallback(DownloadBaseCallback): - def __init__(self, total_pkgs): - self.total_pkgs = total_pkgs - self.downloaded_pkgs = 0 - self.last_pct = 0 - self.last_time = 0 - DownloadBaseCallback.__init__(self) - - def updateProgress(self, name, frac, fread, ftime): - pct = int(frac * 100) - if pct == self.last_pct: - log2("percentage is the same, not updating progress") - return - - self.last_pct = pct - # if run from terminal we can have fancy output - if sys.stdout.isatty(): - sys.stdout.write("\033[sDownloading (%i of %i) %s: %3u%%\033[u" - % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct) - ) - if pct == 100: - #print (_("Downloading (%i of %i) %s: %3u%%") - # % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct) - #) - print (_("Downloading ({0} of {1}) {2}: {3:3}%").format( - self.downloaded_pkgs + 1, self.total_pkgs, name, pct - ) - ) - # but we want machine friendly output when spawned from abrt-server - else: - t = time.time() - if self.last_time == 0: - self.last_time = t - # update only every 5 seconds - if pct == 100 or self.last_time > t or t - self.last_time >= 5: - print (_("Downloading ({0} of {1}) {2}: {3:3}%").format( - self.downloaded_pkgs + 1, self.total_pkgs, name, pct - ) - ) - self.last_time = t - if pct == 100: - self.last_time = 0 - - sys.stdout.flush() - -class DebugInfoDownload(YumBase): - def __init__(self, cache, tmp, keep_rpms=False): - self.cachedir = cache - self.tmpdir = tmp - self.keeprpms = keep_rpms - YumBase.__init__(self) - mute_stdout() - #self.conf.cache = os.geteuid() != 0 - # Setup yum (Ts, RPM db, Repo & Sack) - # doConfigSetup() takes some time, let user know what we are doing - print _("Initializing yum") - try: - # Saw this exception here: - # cannot open Packages index using db3 - Permission denied (13) - # yum.Errors.YumBaseError: Error: rpmdb open failed - self.doConfigSetup() - except Exception, e: - unmute_stdout() - print _("Error initializing yum (YumBase.doConfigSetup): '{0!s}'").format(e) - #return 1 - can't do this in constructor - exit(1) - unmute_stdout() - - # return value will be used as exitcode. So 0 = ok, !0 - error - def download(self, files): - """ @files - """ - installed_size = 0 - total_pkgs = 0 - todownload_size = 0 - downloaded_pkgs = 0 - # nothing to download? - if not files: - return - - #if verbose == 0: - # # this suppress yum messages about setting up repositories - # mute_stdout() - - # make yumdownloader work as non root user - if not self.setCacheDir(): - self.logger.error("Error: can't make cachedir, exiting") - exit(50) - - # disable all not needed - for repo in self.repos.listEnabled(): - repo.close() - self.repos.disableRepo(repo.id) - - # This takes some time, let user know what we are doing - print _("Setting up yum repositories") - # setting-up repos one-by-one, so we can skip the broken ones... - # this helps when users are using 3rd party repos like rpmfusion - # in rawhide it results in: Can't find valid base url... - for r in self.repos.findRepos(pattern="*debuginfo*"): - try: - rid = self.repos.enableRepo(r.id) - self.repos.doSetup(thisrepo=str(r.id)) - log1("enabled repo %s", rid) - setattr(r, "skip_if_unavailable", True) - except Exception, ex: - print _("Can't setup {0}: {1}, disabling").format(r.id, ex) - self.repos.disableRepo(r.id) - - # This is somewhat "magic", it unpacks the metadata making it usable. - # Looks like this is the moment when yum talks to remote servers, - # which takes time (sometimes minutes), let user know why - # we have "paused": - print _("Looking for needed packages in repositories") - try: - self.repos.populateSack(mdtype='metadata', cacheonly=1) - except Exception, e: - print _("Error retrieving metadata: '{0!s}'").format(e) - return 1 - try: - # Saw this exception here: - # raise Errors.NoMoreMirrorsRepoError, errstr - # NoMoreMirrorsRepoError: failure: - # repodata/7e6632b82c91a2e88a66ad848e231f14c48259cbf3a1c3e992a77b1fc0e9d2f6-filelists.sqlite.bz2 - # from fedora-debuginfo: [Errno 256] No more mirrors to try. - self.repos.populateSack(mdtype='filelists', cacheonly=1) - except Exception, e: - print _("Error retrieving filelists: '{0!s}'").format(e) - return 1 - - #if verbose == 0: - # # re-enable the output to stdout - # unmute_stdout() - - not_found = [] - package_files_dict = {} - for debuginfo_path in files: - log2("yum whatprovides %s", debuginfo_path) - pkg = self.pkgSack.searchFiles(debuginfo_path) - # sometimes one file is provided by more rpms, we can use either of - # them, so let's use the first match - if pkg: - if pkg[0] in package_files_dict.keys(): - package_files_dict[pkg[0]].append(debuginfo_path) - else: - package_files_dict[pkg[0]] = [debuginfo_path] - todownload_size += float(pkg[0].size) - installed_size += float(pkg[0].installedsize) - total_pkgs += 1 - - log2("found pkg for %s: %s", debuginfo_path, pkg[0]) - else: - log2("not found pkg for %s", debuginfo_path) - not_found.append(debuginfo_path) - - # connect our progress update callback - dnlcb = MyDownloadCallback(total_pkgs) - self.repos.setProgressBar(dnlcb) - - if verbose != 0 or len(not_found) != 0: - print _("Can't find packages for {0} debuginfo files").format(len(not_found)) - if verbose != 0 or total_pkgs != 0: - print _("Packages to download: {0}").format(total_pkgs) - question = _("Downloading {0:.2f}Mb, installed size: {1:.2f}Mb. Continue?").format( - todownload_size / (1024*1024), - installed_size / (1024*1024) - ) - if noninteractive == False and not reportclient.ask_yes_no(question): - print _("Download cancelled by user") - return RETURN_OK - - for pkg, files in package_files_dict.iteritems(): - dnlcb.downloaded_pkgs = downloaded_pkgs - repo.cache = 0 - remote = pkg.returnSimple('relativepath') - local = os.path.basename(remote) - if not os.path.exists(self.tmpdir): - try: - os.makedirs(self.tmpdir) - except OSError, ex: - print "Can't create tmpdir: %s" % ex - return RETURN_FAILURE - if not os.path.exists(self.cachedir): - try: - os.makedirs(self.cachedir) - except OSError, ex: - print "Can't create cachedir: %s" % ex - return RETURN_FAILURE - local = os.path.join(self.tmpdir, local) - pkg.localpath = local # Hack: to set the localpath we want - err = self.downloadPkgs(pkglist=[pkg]) - # normalize the name - # just str(pkg) doesn't work because it can have epoch - pkg_nvra = pkg.name + "-" + pkg.version + "-" + pkg.release + "." + pkg.arch - package_file_name = pkg_nvra + ".rpm" - if err: - # I observed a zero-length file left on error, - # which prevents cleanup later. Fix it: - try: - os.unlink(self.tmpdir + "/" + package_file_name) - except: - pass - print (_("Downloading package {0} failed").format(pkg)) - else: - unpack_result = unpack_rpm(package_file_name, files, self.tmpdir, - self.cachedir, keeprpms) - if unpack_result == RETURN_FAILURE: - # recursively delete the temp dir on failure - print _("Unpacking failed, aborting download...") - clean_up() - return RETURN_FAILURE - - downloaded_pkgs += 1 - - if not self.keeprpms and os.path.exists(self.tmpdir): - # Was: "All downloaded packages have been extracted, removing..." - # but it was appearing even if no packages were in fact extracted - # (say, when there was one package, and it has download error). - print (_("Removing {0}").format(self.tmpdir)) - try: - os.rmdir(self.tmpdir) - except OSError: - error_msg(_("Can't remove %s, probably contains an error log").format(self.tmpdir)) - -def build_ids_to_path(pfx, build_ids): - """ - build_id1=${build_id:0:2} - build_id2=${build_id:2} - file="usr/lib/debug/.build-id/$build_id1/$build_id2.debug" - """ - return ["%s/usr/lib/debug/.build-id/%s/%s.debug" % (pfx, b_id[:2], b_id[2:]) for b_id in build_ids] - -# beware this finds only missing libraries, but not the executable itself .. - -def filter_installed_debuginfos(build_ids, cache_dirs): - files = build_ids_to_path("", build_ids) - missing = [] - - # 1st pass -> search in /usr/lib - for debuginfo_path in files: - log2("looking: %s", debuginfo_path) - if os.path.exists(debuginfo_path): - log2("found: %s", debuginfo_path) - continue - log2("not found: %s", debuginfo_path) - missing.append(debuginfo_path) - - if missing: - files = missing - missing = [] - else: # nothing is missing, we can stop looking - return missing - - for cache_dir in cache_dirs: - log2("looking in %s" % cache_dir) - for debuginfo_path in files: - cache_debuginfo_path = cache_dir + debuginfo_path - log2("looking: %s", cache_debuginfo_path) - if os.path.exists(cache_debuginfo_path): - log2("found: %s", cache_debuginfo_path) - continue - log2("not found: %s", debuginfo_path) - missing.append(debuginfo_path) - # in next iteration look only for files missing - # from previous iterations - if missing: - files = missing - missing = [] - else: # nothing is missing, we can stop looking - return missing - - return files - -tmpdir = None -def clean_up(): - if tmpdir: - try: - shutil.rmtree(tmpdir) - except OSError, ex: - if ex.errno != errno.ENOENT: - error_msg(_("Can't remove '{0}': {1}").format(tmpdir, ex)) - def sigterm_handler(signum, frame): clean_up() exit(RETURN_OK) @@ -432,6 +66,8 @@ if __name__ == "__main__": keeprpms = False noninteractive = False b_ids = [] + exact_fls = False + missing = None # localization init_gettext() @@ -449,6 +85,7 @@ if __name__ == "__main__": help_text = _( "Usage: %s [-vy] [--ids=BUILD_IDS_FILE]\n" " [--tmpdir=TMPDIR] [--cache=CACHEDIR[:DEBUGINFODIR1:DEBUGINFODIR2...]] [--size_mb=SIZE]\n" + " [-e, --exact=PATH]\n" "\n" "Installs debuginfos for all build-ids listed in BUILD_IDS_FILE\n" "to CACHEDIR, using TMPDIR as temporary staging area.\n" @@ -460,12 +97,13 @@ if __name__ == "__main__": " --tmpdir Default: /tmp/abrt-tmp-debuginfo-RANDOM_SUFFIX\n" " --cache Default: /var/cache/abrt-di\n" " --size_mb Default: 4096\n" + " -e,--exact Download only specified files\n" # --keeprpms is not documented yet because it's a NOP so far ) % PROGNAME try: - opts, args = getopt.getopt(sys.argv[1:], "vyh", - ["help", "ids=", "cache=", "size_mb=", "tmpdir=", "keeprpms"]) + opts, args = getopt.getopt(sys.argv[1:], "vyhe", + ["help", "ids=", "cache=", "size_mb=", "tmpdir=", "keeprpms", "exact="]) except getopt.GetoptError, err: print str(err) # prints something like "option -a not recognized" exit(RETURN_FAILURE) @@ -491,6 +129,10 @@ if __name__ == "__main__": tmpdir = arg elif opt == "--keeprpms": keeprpms = True + # --exact takes precendece over --ids + elif opt in ("-e", "--exact"): + missing=arg.split(':') + exact_fls = True if not cachedirs: try: @@ -516,41 +158,44 @@ if __name__ == "__main__": # for now, we use /tmp... tmpdir = "/tmp/abrt-tmp-debuginfo-%s.%u" % (time.strftime("%Y-%m-%d-%H:%M:%S"), os.getpid()) - fin = sys.stdin - if fbuild_ids != "-": - try: - fin = open(fbuild_ids, "r") - except IOError, ex: - error_msg_and_die(_("Can't open {0}: {1}").format(fbuild_ids, ex)) - for line in fin.readlines(): - b_ids.append(line.strip('\n')) - if not b_ids: - exit(RETURN_FAILURE) + if missing == None: + fin = sys.stdin + if fbuild_ids != "-": + try: + fin = open(fbuild_ids, "r") + except IOError, ex: + error_msg_and_die(_("Can't open {0}: {1}").format(fbuild_ids, ex)) + for line in fin.readlines(): + b_ids.append(line.strip('\n')) + + if not b_ids: + exit(RETURN_FAILURE) + + # Delete oldest/biggest files from cachedir. + # (Note that we need to do it before we check for missing debuginfos) + # + # We can do it as a separate step in report_event.conf, but this + # would require setuid'ing abrt-action-trim-files to abrt:abrt. + # Since we (via abrt-action-install-debuginfo-to-abrt-cache) + # are already running setuid, + # it makes sense to NOT setuid abrt-action-trim-files too, + # but instead run it as our child: + sys.stdout.flush() + try: + pid = os.fork() + if pid == 0: + argv = ["abrt-action-trim-files", "-f", "%um:%s" % (size_mb, cachedirs[0]), "--"] + argv.extend(build_ids_to_path(cachedirs[0], b_ids)) + log2("abrt-action-trim-files %s", argv); + os.execvp("abrt-action-trim-files", argv); + if pid > 0: + os.waitpid(pid, 0); + except Exception, e: + error_msg("Can't execute abrt-action-trim-files: %s", e); - # Delete oldest/biggest files from cachedir. - # (Note that we need to do it before we check for missing debuginfos) - # - # We can do it as a separate step in report_event.conf, but this - # would require setuid'ing abrt-action-trim-files to abrt:abrt. - # Since we (via abrt-action-install-debuginfo-to-abrt-cache) - # are already running setuid, - # it makes sense to NOT setuid abrt-action-trim-files too, - # but instead run it as our child: - sys.stdout.flush() - try: - pid = os.fork() - if pid == 0: - argv = ["abrt-action-trim-files", "-f", "%um:%s" % (size_mb, cachedirs[0]), "--"] - argv.extend(build_ids_to_path(cachedirs[0], b_ids)) - log2("abrt-action-trim-files %s", argv); - os.execvp("abrt-action-trim-files", argv); - if pid > 0: - os.waitpid(pid, 0); - except Exception, e: - error_msg("Can't execute abrt-action-trim-files: %s", e); + missing = filter_installed_debuginfos(b_ids, cachedirs) - missing = filter_installed_debuginfos(b_ids, cachedirs) if missing: log2("%s", missing) print _("Coredump references {0} debuginfo files, {1} of them are not installed").format(len(b_ids), len(missing)) @@ -558,7 +203,7 @@ if __name__ == "__main__": # TODO: should we pass keep_rpms=keeprpms to DebugInfoDownload here?? downloader = DebugInfoDownload(cache=cachedirs[0], tmp=tmpdir) try: - result = downloader.download(missing) + result = downloader.download(missing, download_exact_files=exact_fls) except Exception, ex: error_msg_and_die("Can't download debuginfos: %s", ex) @@ -567,6 +212,6 @@ if __name__ == "__main__": print _("Missing debuginfo file: {0}").format(bid) if not missing: - print _("All {0} debuginfo files are available").format(len(b_ids)) + print _("All debuginfo files are available") exit(RETURN_OK) -- 1.7.7.6
