commit: d7ea5a2a7d0170b6e7aa8c777f1d860828d75107 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Tue May 26 20:56:33 2015 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Tue May 26 20:56:33 2015 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d7ea5a2a
small fixes for diffrent things conf/tbc.conf | 4 +- tbc/bin/tbc_guest_jobs | 7 +- tbc/pym/build_job.py | 7 +- tbc/pym/build_log.py | 9 +-- tbc/pym/jobs.py | 2 +- tbc/pym/package.py | 94 +++++++++++++++++--------- tbc/pym/qachecks.py | 178 +++++++++++++++++++++++++++++++++++++++++++++++++ tbc/pym/sqlquerys.py | 16 +++-- tbc/pym/updatedb.py | 2 +- 9 files changed, 264 insertions(+), 55 deletions(-) diff --git a/conf/tbc.conf b/conf/tbc.conf index ca3c987..339a22a 100644 --- a/conf/tbc.conf +++ b/conf/tbc.conf @@ -8,8 +8,8 @@ SQLHOST=localhost SQLUSER=buildhost # Sql Password SQLPASSWD=buildhost -# TBC git repo name for the setups/configs -TBCGITREPONAME=tinderboxs_configs +# TBC git repo path for the setups/configs /etc/portage +TBCGITREPOPATH=/var/cache/tbc/tinderboxs_configs TBCSONFIG=base # Main logfile LOGFILE=/var/log/tbc diff --git a/tbc/bin/tbc_guest_jobs b/tbc/bin/tbc_guest_jobs index 8cecba2..a6fa566 100755 --- a/tbc/bin/tbc_guest_jobs +++ b/tbc/bin/tbc_guest_jobs @@ -4,7 +4,7 @@ # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function -from tbc.readconf import get_conf_settings +from tbc.readconf import read_config_settings from tbc.ConnectionManager import NewConnection from tbc.sqlquerys import add_tbc_logs, get_config_id, check_host_updatedb, update_deamon_status from tbc.check_setup import check_configure_guest @@ -18,8 +18,7 @@ import time def main(): repeat = True - reader = get_conf_settings() - tbc_settings_dict=reader.read_tbc_settings_all() + tbc_settings_dict = read_config_settings() config = tbc_settings_dict['tbc_config'] hostname = tbc_settings_dict['hostname'] Session = sessionmaker(bind=NewConnection(tbc_settings_dict)) @@ -29,7 +28,7 @@ def main(): init_build_job = build_job_action(config_id, session) while repeat: jobs_main(session, config) - if not check_configure_guest(session, config_id) or check_host_updatedb(session): + if not check_configure_guest(session, tbc_settings_dict, config_id) or check_host_updatedb(session): time.sleep(30) continue else: diff --git a/tbc/pym/build_job.py b/tbc/pym/build_job.py index 0167882..dfe6639 100644 --- a/tbc/pym/build_job.py +++ b/tbc/pym/build_job.py @@ -12,9 +12,9 @@ from portage import _encodings from portage import _unicode_decode from portage.versions import cpv_getkey from portage.dep import check_required_use -from tbc.manifest import tbc_manifest from tbc.depclean import do_depclean from tbc.flags import tbc_use_flags +from tbc.qacheck import check_file_in_manifest from tbc.main import emerge_main from tbc.build_log import log_fail_queru from tbc.actions import load_emerge_config @@ -32,20 +32,19 @@ class build_job_action(object): package = build_dict['package'] cpv = build_dict['cpv'] pkgdir = portdb.getRepositoryPath(repo) + "/" + cp - init_manifest = tbc_manifest(settings, pkgdir) build_use_flags_list = [] try: ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + build_dict['ebuild_version'] + ".ebuild")[0] except: ebuild_version_checksum_tree = None if ebuild_version_checksum_tree == build_dict['checksum']: - manifest_error = init_manifest.check_file_in_manifest(portdb, cpv, build_use_flags_list, repo) + manifest_error = check_file_in_manifest(pkgdir, settings, portdb, cpv, build_use_flags_list, repo) if manifest_error is None: init_flags = tbc_use_flags(settings, portdb, cpv) build_use_flags_list = init_flags.comper_useflags(build_dict) log_msg = "build_use_flags_list %s" % (build_use_flags_list,) add_tbc_logs(self._session, log_msg, "info", self._config_id) - manifest_error = init_manifest.check_file_in_manifest(portdb, cpv, build_use_flags_list, repo) + manifest_error = check_file_in_manifest(pkgdir, settings, portdb, cpv, build_use_flags_list, repo) if manifest_error is None: build_dict['check_fail'] = False build_cpv_dict = {} diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py index f6ceea6..26735e2 100644 --- a/tbc/pym/build_log.py +++ b/tbc/pym/build_log.py @@ -20,7 +20,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'tbc.actions:action_info,load_emerge_config', ) -from tbc.repoman_tbc import tbc_repoman +from tbc.qachecks import check_repoman from tbc.text import get_log_text_dict from tbc.package import tbc_package from tbc.readconf import read_config_settings @@ -184,7 +184,6 @@ def search_buildlog(session, logfile_text_dict, max_text_lines): def get_buildlog_info(session, settings, pkg, build_dict): myportdb = portage.portdbapi(mysettings=settings) - init_repoman = tbc_repoman(settings, myportdb) logfile_text_dict, max_text_lines = get_log_text_dict(settings.get("PORTAGE_LOG_FILE")) hilight_dict = search_buildlog(session, logfile_text_dict, max_text_lines) error_log_list = [] @@ -206,11 +205,9 @@ def get_buildlog_info(session, settings, pkg, build_dict): i = i +1 # Run repoman check_repoman() - repoman_error_list = init_repoman.check_repoman(build_dict['cpv'], pkg.repo) - if repoman_error_list != []: + repoman_error_list = check_repoman(settings, myportdb, build_dict['cpv'], pkg.repo) + if repoman_error_list: sum_build_log_list.append("1") # repoman = 1 - else: - repoman_error_list = False if qa_error_list != []: sum_build_log_list.append("2") # qa = 2 else: diff --git a/tbc/pym/jobs.py b/tbc/pym/jobs.py index a805a22..830aa07 100644 --- a/tbc/pym/jobs.py +++ b/tbc/pym/jobs.py @@ -59,7 +59,7 @@ def jobs_main(session, config_id): update_job_list(session, "Runing", job_id) log_msg = "Job %s is runing." % (job_id,) add_tbc_logs(session, log_msg, "info", config_id) - if update_db_main(session, None, config_id): + if update_db_main(session, None, config_id): update_job_list(session, "Done", job_id) log_msg = "Job %s is done.." % (job_id,) add_tbc_logs(session, log_msg, "info", config_id) diff --git a/tbc/pym/package.py b/tbc/pym/package.py index 5f172cb..a61a1c9 100644 --- a/tbc/pym/package.py +++ b/tbc/pym/package.py @@ -8,12 +8,13 @@ from tbc.flags import tbc_use_flags from tbc.manifest import tbc_manifest from tbc.text import get_ebuild_cvs_revision from tbc.flags import tbc_use_flags +from tbc.qachecks import digestcheck, check_repoman from tbc.sqlquerys import add_tbc_logs, get_package_info, get_config_info, \ add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \ get_package_metadata_sql, update_package_metadata, update_manifest_sql, \ get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \ - get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info -from tbc.readconf import get_conf_settings + get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info, \ + get_ebuild_info_ebuild_id class tbc_package(object): @@ -219,6 +220,30 @@ class tbc_package(object): # Add the ebuild to the build jobs table if needed self.add_new_build_job_db(ebuild_id_list, packageDict, config_cpv_listDict) + def get_manifest_checksum_tree(self, pkgdir, cp, repo, mytree): + + # Get the cp manifest file checksum. + try: + manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0] + except: + # Get the package list from the repo + package_list_tree =self. _myportdb.cp_all(trees=mytree) + if cp in package_list_tree: + log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,) + add_zobcs_logs(self._session, log_msg, "error", self._config_id) + log_msg = "C %s:%s ... Fail." % (cp, repo) + add_zobcs_logs(self._session, log_msg, "error", self._config_id) + return "0" + fail_msg = digestcheck(self._mysettings, pkgdir) + if fail_msg: + log_msg = "QA: Manifest file has errors. :%s:%s" % (cp, repo,) + add_zobcs_logs(self._session, log_msg, "error", self._config_id) + add_zobcs_logs(self._session, fail_msg, "error", self._config_id) + log_msg = "C %s:%s ... Fail." % (cp, repo) + add_zobcs_logs(self._session, log_msg, "error", self._config_id) + return None + return manifest_checksum_tree + def add_new_package_db(self, cp, repo): # Add new categories package ebuild to tables package and ebuilds # C = Checking @@ -230,16 +255,9 @@ class tbc_package(object): repodir = self._myportdb.getRepositoryPath(repo) pkgdir = repodir + "/" + cp # Get RepoDIR + cp - # Get the cp manifest file checksum. - try: - manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0] - except: - manifest_checksum_tree = "0" - log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,) - add_tbc_logs(self._session, log_msg, "info", self._config_id) - log_msg = "C %s:%s ... Fail." % (cp, repo) - add_tbc_logs(self._session, log_msg, "info", self._config_id) - return None + manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree) + if manifest_checksum_tree is None: + return package_id = add_new_package_sql(self._session, cp, repo) package_metadataDict = self.get_package_metadataDict(pkgdir, package_id) @@ -259,6 +277,10 @@ class tbc_package(object): new_ebuild_id_list = [] old_ebuild_id_list = [] for cpv in sorted(ebuild_list_tree): + repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo) + if repoman_fail: + log_msg = "Repoman %s:%s ... Fail." % (cpv, repo) + add_tbc_logs(self._session, log_msg, "error", self._config_id) packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo) self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree) @@ -275,17 +297,11 @@ class tbc_package(object): add_tbc_logs(self._session, log_msg, "info", self._config_id) repodir = self._myportdb.getRepositoryPath(repo) pkgdir = repodir + "/" + cp # Get RepoDIR + cp - - # Get the cp mainfest file checksum - try: - manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0] - except: - manifest_checksum_tree = "0" - log_msg = "QA: Can't checksum the Manifest file. %s:%s" % (cp, repo,) - add_tbc_logs(self._session, log_msg, "info", self._config_id) - log_msg = "C %s:%s ... Fail." % (cp, repo) - add_tbc_logs(self._session, log_msg, "info", self._config_id) - return None + mytree = [] + mytree.append(repodir) + manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree) + if manifest_checksum_tree is None: + return # if we NOT have the same checksum in the db update the package if manifest_checksum_tree != PackageInfo.Checksum: @@ -295,18 +311,28 @@ class tbc_package(object): add_tbc_logs(self._session, log_msg, "info", self._config_id) # Get the ebuild list for cp - mytree = [] - mytree.append(repodir) + old_ebuild_id_list = [] ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree) if ebuild_list_tree == []: - log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,) - add_tbc_logs(self._session, log_msg, "info", self._config_id) - log_msg = "C %s:%s ... Fail." % (cp, repo) - add_tbc_logs(self._session, log_msg, "info", self._config_id) - return None + if manifest_checksum_tree == "0": + old_ebuild_id_list = get_ebuild_id_list(self._session, package_id) + for ebuild_id in old_ebuild_id_list: + EbuildInfo = get_ebuild_info_ebuild_id(self._session, ebuild_id) + cpv = cp + "-" + EbuildInfo.Version + # R = remove ebuild + log_msg = "R %s:%s" % (cpv, repo,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + add_old_ebuild(session, old_ebuild_id_list) + log_msg = "C %s:%s ... Done." % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + else: + log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + log_msg = "C %s:%s ... Fail." % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + return packageDict ={} new_ebuild_id_list = [] - old_ebuild_id_list = [] for cpv in sorted(ebuild_list_tree): # split out ebuild version @@ -334,6 +360,12 @@ class tbc_package(object): else: ebuild_version_manifest_checksum_db = checksums_db + if ebuild_version_manifest_checksum_db is None or ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db: + repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo) + if repoman_fail: + log_msg = "Repoman %s:%s ... Fail." % (cpv, repo) + add_zobcs_logs(self._session, log_msg, "info", self._config_id) + # Check if the checksum have change if ebuild_version_manifest_checksum_db is None: # N = New ebuild diff --git a/tbc/pym/qachecks.py b/tbc/pym/qachecks.py new file mode 100644 index 0000000..f1d0fb1 --- /dev/null +++ b/tbc/pym/qachecks.py @@ -0,0 +1,178 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import os +import warnings +import sys +import codecs +from portage import os, _encodings, _unicode_decode +from portage.exception import DigestException, FileNotFound +from portage.localization import _ +from portage.manifest import Manifest +from portage import os, _encodings, _unicode_decode, _unicode_encode +from portage.exception import DigestException, FileNotFound, ParseError, PermissionDenied +from _emerge.Package import Package +from _emerge.RootConfig import RootConfig +from repoman.checks import run_checks +import portage + +# Copy of portage.digestcheck() but without the writemsg() stuff +def digestcheck(mysettings, pkgdir): + """ + Verifies checksums. Assumes all files have been downloaded. + @rtype: int + @return: 1 on success and 0 on failure + """ + + myfiles = [] + justmanifest = None + mf = None + + if mysettings.get("EBUILD_SKIP_MANIFEST") == "1": + return False + hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", "")) + if hash_filter.transparent: + hash_filter = None + if mf is None: + mf = mysettings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))) + mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"]) + try: + if not mf.thin and strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings: + if mf.fhashdict.get("EBUILD"): + mf.checkTypeHashes("EBUILD", hash_filter=hash_filter) + if mf.fhashdict.get("AUX"): + mf.checkTypeHashes("AUX", hash_filter=hash_filter) + if mf.fhashdict.get("MISC"): + mf.checkTypeHashes("MISC", ignoreMissingFiles=True, + hash_filter=hash_filter) + for f in myfiles: + ftype = mf.findFile(f) + if ftype is None: + if mf.allow_missing: + continue + return ("\n!!! Missing digest for '%s'\n") % f + mf.checkFileHashes(ftype, f, hash_filter=hash_filter) + except FileNotFound as e: + return ("\n!!! A file listed in the Manifest could not be found: %s\n") % str(e) + except DigestException as e: + return ("!!! Digest verification failed: %s\nReason: %s\nGot: %s\nExpected: %s") \ + % (e.value[0], e.value[1], e.value[2], e.value[3]) + if mf.thin or mf.allow_missing: + # In this case we ignore any missing digests that + # would otherwise be detected below. + return False + # Make sure that all of the ebuilds are actually listed in the Manifest. + for f in os.listdir(pkgdir): + pf = None + if f[-7:] == '.ebuild': + pf = f[:-7] + if pf is not None and not mf.hasFile("EBUILD", f): + return ("!!! A file is not listed in the Manifest: '%s'\n") % os.path.join(pkgdir, f) + + # epatch will just grab all the patches out of a directory, so we have to + # make sure there aren't any foreign files that it might grab. + filesdir = os.path.join(pkgdir, "files") + + for parent, dirs, files in os.walk(filesdir): + try: + parent = _unicode_decode(parent, + encoding=_encodings['fs'], errors='strict') + except UnicodeDecodeError: + parent = _unicode_decode(parent, + encoding=_encodings['fs'], errors='replace') + return ("!!! Path contains invalid character(s) for encoding '%s': '%s'") % (_encodings['fs'], parent) + for d in dirs: + d_bytes = d + try: + d = _unicode_decode(d, + encoding=_encodings['fs'], errors='strict') + except UnicodeDecodeError: + d = _unicode_decode(d, + encoding=_encodings['fs'], errors='replace') + return ("!!! Path contains invalid character(s) for encoding '%s': '%s'") % (_encodings['fs'], os.path.join(parent, d)) + if d.startswith(".") or d == "CVS": + dirs.remove(d_bytes) + for f in files: + try: + f = _unicode_decode(f, + encoding=_encodings['fs'], errors='strict') + except UnicodeDecodeError: + f = _unicode_decode(f, + encoding=_encodings['fs'], errors='replace') + if f.startswith("."): + continue + f = os.path.join(parent, f)[len(filesdir) + 1:] + return ("!!! File name contains invalid character(s) for encoding '%s': '%s'") % (_encodings['fs'], f) + if f.startswith("."): + continue + f = os.path.join(parent, f)[len(filesdir) + 1:] + file_type = mf.findFile(f) + if file_type != "AUX" and not f.startswith("digest-"): + return ("!!! A file is not listed in the Manifest: '%s'\n") % os.path.join(filesdir, f) + return False + + +def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list, repo): + myfetchlistdict = portage.FetchlistDict(pkgdir, mysettings, portdb) + my_manifest = portage.Manifest(pkgdir, mysettings['DISTDIR'], fetchlist_dict=myfetchlistdict, manifest1_compat=False, from_scratch=False) + ebuild_version = portage.versions.cpv_getversion(cpv) + package = portage.versions.cpv_getkey(cpv).split("/")[1] + if my_manifest.findFile(package + "-" + ebuild_version + ".ebuild") is None: + return "Ebuild file not found." + tree = portdb.getRepositoryPath(repo) + cpv_fetchmap = portdb.getFetchMap(cpv, useflags=build_use_flags_list, mytree=tree) + self._mysettings.unlock() + try: + portage.fetch(cpv_fetchmap, mysettings, listonly=0, fetchonly=0, locks_in_subdir='.locks', use_locks=1, try_mirrors=1) + except: + self._mysettings.lock() + return "Can't fetch the file." + finally: + self._mysettings.lock() + try: + my_manifest.checkCpvHashes(cpv, checkDistfiles=True, onlyDistfiles=False, checkMiscfiles=True) + except: + return "Can't fetch the file or the hash failed." + try: + portdb.fetch_check(cpv, useflags=build_use_flags_list, mysettings=mysettings, all=False) + except: + return "Fetch check failed." + return + +def check_repoman(mysettings, myportdb, cpv, repo): + # We run repoman run_checks on the ebuild + ebuild_version_tree = portage.versions.cpv_getversion(cpv) + element = portage.versions.cpv_getkey(cpv).split('/') + categories = element[0] + package = element[1] + pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package + full_path = pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild" + root = '/' + trees = { + root : {'porttree' : portage.portagetree(root, settings=mysettings)} + } + root_config = RootConfig(mysettings, trees[root], None) + allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_")) + allvars.update(Package.metadata_keys) + allvars = sorted(allvars) + myaux = dict(zip(allvars, myportdb.aux_get(cpv, allvars, myrepo=repo))) + pkg = Package(cpv=cpv, metadata=myaux, root_config=root_config, type_name='ebuild') + fails = [] + try: + # All ebuilds should have utf_8 encoding. + f = codecs.open(_unicode_encode(full_path, + encoding = _encodings['fs'], errors = 'strict'), + mode = 'r', encoding = _encodings['repo.content']) + try: + for check_name, e in run_checks(f, pkg): + fails.append(check_name + ": " + e) + finally: + f.close() + except UnicodeDecodeError: + # A file.UTF8 failure will have already been recorded above. + pass + # fails will have a list with repoman errors + if fails == []: + return False + return fails \ No newline at end of file diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py index f9fc8b4..097ded2 100644 --- a/tbc/pym/sqlquerys.py +++ b/tbc/pym/sqlquerys.py @@ -147,6 +147,9 @@ def get_ebuild_info(session, build_dict): return EbuildInfo.all(), True return EbuildInfo2, False +def get_ebuild_info_ebuild_id(session, ebuild_id): + return session.query(Ebuilds).filter_by(EbuildId = ebuild_id).filter_by(Active = True).one() + def get_build_job_id(session, build_dict): BuildJobsIdInfo = session.query(BuildJobs.BuildJobId).filter_by(EbuildId = build_dict['ebuild_id']).filter_by(ConfigId = build_dict['config_id']).all() if BuildJobsIdInfo == []: @@ -500,10 +503,6 @@ def get_ebuild_checksums(session, package_id, ebuild_version): except MultipleResultsFound as e: EbuildInfo2 = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Version = ebuild_version).filter_by(Active = True).all() for Ebuild in EbuildInfo2: - print("ebuild version checksum") - print(ebuild_version) - print(Ebuild.Version) - print(Ebuild.Checksum) ebuild_checksum_list.append(Ebuild.Checksum) return ebuild_checksum_list, True return EbuildInfo.Checksum, False @@ -522,8 +521,13 @@ def get_ebuild_id_db(session, checksum, package_id): return EbuildInfos.EbuildId, False def check_host_updatedb(session): + jobs = False try: JobsInfo = session.query(Jobs).filter_by(Status = 'Done').filter_by(JobType = 'esync').one() except NoResultFound as e: - return True - return False + jobs = True + try: + JobsInfo = session.query(Jobs).filter_by(Status = 'Done').filter_by(JobType = 'updatedb').one() + except NoResultFound as e: + jobs = True + return jobs diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py index 3cbe495..9b74093 100644 --- a/tbc/pym/updatedb.py +++ b/tbc/pym/updatedb.py @@ -133,7 +133,7 @@ def update_cpv_db(session, config_id, tbc_settings_dict): def update_db_main(session, repo_cp_dict, config_id): # Main - if repo_cp_dict == {}: + if repo_cp_dict == {}: return True # Logging tbc_settings_dict = reader. read_config_settings()