commit: c78602c4fc3043be4c0c722f0b5886a372ad9382 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Sat Apr 18 20:26:35 2015 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Sat Apr 18 20:26:35 2015 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c78602c4
add code/files in tbc/pym for the backend python code tbc/pym/ConnectionManager.py | 22 ++ tbc/pym/__init__.py | 0 tbc/pym/build_depgraph.py | 65 ++++++ tbc/pym/build_job.py | 166 ++++++++++++++ tbc/pym/build_log.py | 336 ++++++++++++++++++++++++++++ tbc/pym/buildquerydb.py | 102 +++++++++ tbc/pym/check_setup.py | 74 +++++++ tbc/pym/db_mapping.py | 260 ++++++++++++++++++++++ tbc/pym/depclean.py | 53 +++++ tbc/pym/flags.py | 231 +++++++++++++++++++ tbc/pym/jobs.py | 86 ++++++++ tbc/pym/old_cpv.py | 97 ++++++++ tbc/pym/package.py | 355 ++++++++++++++++++++++++++++++ tbc/pym/readconf.py | 58 +++++ tbc/pym/sqlquerys.py | 512 +++++++++++++++++++++++++++++++++++++++++++ tbc/pym/sync.py | 73 ++++++ tbc/pym/text.py | 49 +++++ tbc/pym/updatedb.py | 135 ++++++++++++ 18 files changed, 2674 insertions(+) diff --git a/tbc/pym/ConnectionManager.py b/tbc/pym/ConnectionManager.py new file mode 100644 index 0000000..40abfd5 --- /dev/null +++ b/tbc/pym/ConnectionManager.py @@ -0,0 +1,22 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +import logging + +def NewConnection(tbc_settings_dict): + backend=tbc_settings_dict['sql_backend'] + host=tbc_settings_dict['sql_host'] + user=tbc_settings_dict['sql_user'] + password=tbc_settings_dict['sql_passwd'] + database=tbc_settings_dict['sql_db'] + if backend == 'mysql': + try: + from sqlalchemy import create_engine + except ImportError: + print("Please install a recent version of dev-python/sqlalchemy for Python") + sys.exit(1) + #logging.basicConfig() + #logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + mysqldriver = 'mysql+mysqlconnector' + return create_engine(mysqldriver + '://' + user + ':' + password + '@' + host + '/' + database, pool_recycle=120) diff --git a/tbc/pym/__init__.py b/tbc/pym/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tbc/pym/build_depgraph.py b/tbc/pym/build_depgraph.py new file mode 100644 index 0000000..812e576 --- /dev/null +++ b/tbc/pym/build_depgraph.py @@ -0,0 +1,65 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +from _emerge.create_depgraph_params import create_depgraph_params +from _emerge.depgraph import backtrack_depgraph +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'tbc.actions:load_emerge_config', +) +from portage.exception import PackageSetNotFound + +from tbc.build_log import log_fail_queru + +def build_mydepgraph(settings, trees, mtimedb, myopts, myparams, myaction, myfiles, spinner, build_dict, session): + try: + success, mydepgraph, favorites = backtrack_depgraph( + settings, trees, myopts, myparams, myaction, myfiles, spinner) + except portage.exception.PackageSetNotFound as e: + root_config = trees[settings["ROOT"]]["root_config"] + display_missing_pkg_set(root_config, e.value) + build_dict['type_fail'] = "depgraph fail" + build_dict['check_fail'] = True + else: + if not success: + repeat = True + repeat_times = 0 + while repeat: + if mydepgraph._dynamic_config._needed_p_mask_changes: + build_dict['type_fail'] = "Mask package or dep" + build_dict['check_fail'] = True + elif mydepgraph._dynamic_config._needed_use_config_changes: + mydepgraph._display_autounmask() + build_dict['type_fail'] = "Need use change" + build_dict['check_fail'] = True + elif mydepgraph._dynamic_config._slot_conflict_handler: + build_dict['type_fail'] = "Slot blocking" + build_dict['check_fail'] = True + elif mydepgraph._dynamic_config._circular_deps_for_display: + build_dict['type_fail'] = "Circular Deps" + build_dict['check_fail'] = True + elif mydepgraph._dynamic_config._unsolvable_blockers: + build_dict['type_fail'] = "Blocking packages" + build_dict['check_fail'] = True + else: + build_dict['type_fail'] = "Dep calc fail" + build_dict['check_fail'] = True + mydepgraph.display_problems() + if repeat_times is 2: + repeat = False + log_fail_queru(session, build_dict, settings) + else: + repeat_times = repeat_times + 1 + settings, trees, mtimedb = load_emerge_config() + myparams = create_depgraph_params(myopts, myaction) + try: + success, mydepgraph, favorites = backtrack_depgraph( + settings, trees, myopts, myparams, myaction, myfiles, spinner) + except portage.exception.PackageSetNotFound as e: + root_config = trees[settings["ROOT"]]["root_config"] + display_missing_pkg_set(root_config, e.value) + if success: + repeat = False + + return success, settings, trees, mtimedb, mydepgraph diff --git a/tbc/pym/build_job.py b/tbc/pym/build_job.py new file mode 100644 index 0000000..0167882 --- /dev/null +++ b/tbc/pym/build_job.py @@ -0,0 +1,166 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +import portage +import os +import re +import sys +import signal + +from portage import _encodings +from portage import _unicode_decode +from portage.versions import cpv_getkey +from portage.dep import check_required_use +from tbc.manifest import tbc_manifest +from tbc.depclean import do_depclean +from tbc.flags import tbc_use_flags +from tbc.main import emerge_main +from tbc.build_log import log_fail_queru +from tbc.actions import load_emerge_config +from tbc.sqlquerys import add_tbc_logs, get_packages_to_build, update_buildjobs_status, is_build_job_done + +class build_job_action(object): + + def __init__(self, config_id, session): + self._config_id = config_id + self._session = session + + def make_build_list(self, build_dict, settings, portdb): + cp = build_dict['cp'] + repo = build_dict['repo'] + package = build_dict['package'] + cpv = build_dict['cpv'] + pkgdir = portdb.getRepositoryPath(repo) + "/" + cp + init_manifest = tbc_manifest(settings, pkgdir) + build_use_flags_list = [] + try: + ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + build_dict['ebuild_version'] + ".ebuild")[0] + except: + ebuild_version_checksum_tree = None + if ebuild_version_checksum_tree == build_dict['checksum']: + manifest_error = init_manifest.check_file_in_manifest(portdb, cpv, build_use_flags_list, repo) + if manifest_error is None: + init_flags = tbc_use_flags(settings, portdb, cpv) + build_use_flags_list = init_flags.comper_useflags(build_dict) + log_msg = "build_use_flags_list %s" % (build_use_flags_list,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + manifest_error = init_manifest.check_file_in_manifest(portdb, cpv, build_use_flags_list, repo) + if manifest_error is None: + build_dict['check_fail'] = False + build_cpv_dict = {} + build_cpv_dict[cpv] = build_use_flags_list + log_msg = "build_cpv_dict: %s" % (build_cpv_dict,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + return build_cpv_dict + build_dict['type_fail'] = "Manifest error" + build_dict['check_fail'] = True + log_msg = "Manifest error: %s:%s" % (cpv, manifest_error) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + else: + build_dict['type_fail'] = "Wrong ebuild checksum" + build_dict['check_fail'] = True + if build_dict['check_fail'] is True: + log_fail_queru(self._session, build_dict, settings) + return None + + def build_procces(self, buildqueru_cpv_dict, build_dict, settings, portdb): + build_cpv_list = [] + depclean_fail = True + for k, build_use_flags_list in buildqueru_cpv_dict.items(): + build_cpv_list.append("=" + k) + if not build_use_flags_list == None: + build_use_flags = "" + for flags in build_use_flags_list: + build_use_flags = build_use_flags + flags + " " + filetext = '=' + k + ' ' + build_use_flags + log_msg = "filetext: %s" % filetext + add_tbc_logs(self._session, log_msg, "info", self._config_id) + with open("/etc/portage/package.use/99_autounmask", "a") as f: + f.write(filetext) + f.write('\n') + f.close + log_msg = "build_cpv_list: %s" % (build_cpv_list,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + + # We remove the binary package if removebin is true + if build_dict['removebin']: + package = build_dict['package'] + pv = package + "-" + build_dict['ebuild_version'] + binfile = settings['PKGDIR'] + "/" + build_dict['category'] + "/" + pv + ".tbz2" + try: + os.remove(binfile) + except: + log_msg = "Binary file was not removed or found: %s" % (binfile,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + + argscmd = [] + for emerge_option in build_dict['emerge_options']: + if emerge_option == '--depclean': + pass + elif emerge_option == '--nodepclean': + pass + elif emerge_option == '--nooneshot': + pass + else: + if not emerge_option in argscmd: + argscmd.append(emerge_option) + for build_cpv in build_cpv_list: + argscmd.append(build_cpv) + print("Emerge options: %s" % argscmd) + log_msg = "argscmd: %s" % (argscmd,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + + # Call main_emerge to build the package in build_cpv_list + print("Build: %s" % build_dict) + update_buildjobs_status(self._session, build_dict['build_job_id'], 'Building', self._config_id) + build_fail = emerge_main(argscmd, build_dict, self._session) + # Run depclean + if '--depclean' in build_dict['emerge_options'] and not '--nodepclean' in build_dict['emerge_options']: + depclean_fail = do_depclean() + try: + os.remove("/etc/portage/package.use/99_autounmask") + with open("/etc/portage/package.use/99_autounmask", "a") as f: + f.close + except: + pass + + if is_build_job_done(self._session, build_dict['build_job_id']): + update_buildjobs_status(self._session, build_dict['build_job_id'], 'Looked', self._config_id) + log_msg = "build_job %s was not removed" % (build_dict['build_job_id'],) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + print("qurery was not removed") + build_dict['type_fail'] = "Querey was not removed" + build_dict['check_fail'] = True + log_fail_queru(self._session, build_dict, settings) + if build_fail is True: + build_dict['type_fail'] = "Emerge faild" + build_dict['check_fail'] = True + log_msg = "Emerge faild!" + add_tbc_logs(self._session, log_msg, "info", self._config_id) + return True + return False + + def procces_build_jobs(self): + build_dict = {} + build_dict = get_packages_to_build(self._session, self._config_id) + if build_dict is None: + return + print("build_dict: %s" % (build_dict,)) + log_msg = "build_dict: %s" % (build_dict,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + if not build_dict['ebuild_id'] is None and build_dict['checksum'] is not None: + settings, trees, mtimedb = load_emerge_config() + portdb = trees[settings["ROOT"]]["porttree"].dbapi + buildqueru_cpv_dict = self.make_build_list(build_dict, settings, portdb) + log_msg = "buildqueru_cpv_dict: %s" % (buildqueru_cpv_dict,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + if buildqueru_cpv_dict is None: + return + fail_build_procces = self.build_procces(buildqueru_cpv_dict, build_dict, settings, portdb) + return + if not build_dict['emerge_options'] is [] and build_dict['ebuild_id'] is None: + return + if not build_dict['ebuild_id'] is None and build_dict['emerge_options'] is None: + pass + # del_old_queue(self._session, build_dict['queue_id']) diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py new file mode 100644 index 0000000..a7d5314 --- /dev/null +++ b/tbc/pym/build_log.py @@ -0,0 +1,336 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +import re +import os +import platform +import hashlib + +from portage.versions import catpkgsplit, cpv_getversion +import portage +from portage.util import writemsg, \ + writemsg_level, writemsg_stdout +from portage import _encodings +from portage import _unicode_encode + +from _emerge.main import parse_opts + +portage.proxy.lazyimport.lazyimport(globals(), + 'tbc.actions:action_info,load_emerge_config', +) + +from tbc.repoman_tbc import tbc_repoman +from tbc.text import get_log_text_dict +from tbc.package import tbc_package +from tbc.readconf import get_conf_settings +from tbc.flags import tbc_use_flags +from tbc.ConnectionManager import NewConnection +from tbc.sqlquerys import add_tbc_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \ + get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \ + add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \ + update_buildjobs_status +from sqlalchemy.orm import sessionmaker + +def get_build_dict_db(session, config_id, settings, pkg): + myportdb = portage.portdbapi(mysettings=settings) + cpvr_list = catpkgsplit(pkg.cpv, silent=1) + categories = cpvr_list[0] + package = cpvr_list[1] + repo = pkg.repo + ebuild_version = cpv_getversion(pkg.cpv) + log_msg = "Logging %s:%s" % (pkg.cpv, repo,) + add_tbc_logs(session, log_msg, "info", config_id) + PackageInfo = get_package_info(session, categories, package, repo) + build_dict = {} + build_dict['ebuild_version'] = ebuild_version + build_dict['package_id'] = PackageInfo.PackageId + build_dict['cpv'] = pkg.cpv + build_dict['categories'] = categories + build_dict['package'] = package + build_dict['config_id'] = config_id + init_useflags = tbc_use_flags(settings, myportdb, pkg.cpv) + iuse_flags_list, final_use_list = init_useflags.get_flags_pkg(pkg, settings) + iuse = [] + for iuse_line in iuse_flags_list: + iuse.append(init_useflags.reduce_flag(iuse_line)) + iuse_flags_list2 = list(set(iuse)) + use_enable = final_use_list + use_disable = list(set(iuse_flags_list2).difference(set(use_enable))) + use_flagsDict = {} + for x in use_enable: + use_id = get_use_id(session, x) + use_flagsDict[use_id] = True + for x in use_disable: + use_id = get_use_id(session, x) + use_flagsDict[use_id] = False + if use_enable == [] and use_disable == []: + build_dict['build_useflags'] = None + else: + build_dict['build_useflags'] = use_flagsDict + pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package + ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir+ "/" + package + "-" + ebuild_version + ".ebuild")[0] + build_dict['checksum'] = ebuild_version_checksum_tree + ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id']) + if status: + if ebuild_id_list is None: + log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,) + add_tbc_logs(session, log_msg, "info", config_id) + update_manifest_sql(session, build_dict['package_id'], "0") + init_package = tbc_package(session, settings, myportdb) + init_package.update_package_db(build_dict['package_id']) + ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id']) + if status and ebuild_id is None: + log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,) + add_tbc_logs(session, log_msg, "error", config_id) + else: + old_ebuild_id_list = [] + for ebuild_id in ebuild_id_list: + log_msg = "%s:%s:%s Dups of checksums" % (pkg.cpv, repo, ebuild_id,) + add_tbc_logs(session, log_msg, "error", config_id) + old_ebuild_id_list.append(ebuild_id) + add_old_ebuild(session, old_ebuild_id_list) + return + build_dict['ebuild_id'] = ebuild_id_list + + build_job_id = get_build_job_id(session, build_dict) + if build_job_id is None: + build_dict['build_job_id'] = None + else: + build_dict['build_job_id'] = build_job_id + return build_dict + +def search_buildlog(session, logfile_text_dict, max_text_lines): + log_search_list = get_hilight_info(session) + hilight_list = [] + for index, text_line in logfile_text_dict.items(): + for search_pattern in log_search_list: + if re.search(search_pattern.HiLightSearch, text_line): + hilight_tmp = {} + hilight_tmp['startline'] = index - search_pattern.HiLightStart + hilight_tmp['hilight'] = search_pattern.HiLightCssId + if search_pattern.HiLightSearchEnd == "": + hilight_tmp['endline'] = index + search_pattern.HiLightEnd + if hilight_tmp['endline'] > max_text_lines: + hilight_tmp['endline'] = max_text_lines + elif not search_pattern.HiLightSearchEnd == "" and (index + 1) >= max_text_lines: + hilight_tmp['endline'] = max_text_lines + else: + i = index + 1 + match = True + while match: + if i >= max_text_lines: + match = False + break + if re.search(search_pattern.HiLightSearchPattern, logfile_text_dict[i]) and re.search(search_pattern.HiLightSearchPattern, logfile_text_dict[i + 1]): + for search_pattern2 in log_search_list: + if re.search(search_pattern2.HiLightSearch, logfile_text_dict[i]): + match = False + if match: + i = i + 1 + elif re.search(search_pattern.HiLightSearchPattern, logfile_text_dict[i]) and re.search(search_pattern.HiLightSearchEnd, logfile_text_dict[i + 1]): + i = i + 1 + match = False + else: + match = False + if i >= max_text_lines: + hilight_tmp['endline'] = max_text_lines + if re.search(search_pattern.HiLightSearchEnd, logfile_text_dict[i]): + hilight_tmp['endline'] = i + else: + hilight_tmp['endline'] = i - 1 + hilight_list.append(hilight_tmp) + + new_hilight_dict = {} + for hilight_tmp in hilight_list: + add_new_hilight = True + add_new_hilight_middel = None + for k, v in sorted(new_hilight_dict.items()): + if hilight_tmp['startline'] == hilight_tmp['endline']: + if v['endline'] == hilight_tmp['startline'] or v['startline'] == hilight_tmp['startline']: + add_new_hilight = False + if hilight_tmp['startline'] > v['startline'] and hilight_tmp['startline'] < v['endline']: + add_new_hilight = False + add_new_hilight_middel = k + else: + if v['endline'] == hilight_tmp['startline'] or v['startline'] == hilight_tmp['startline']: + add_new_hilight = False + if hilight_tmp['startline'] > v['startline'] and hilight_tmp['startline'] < v['endline']: + add_new_hilight = False + if add_new_hilight is True: + adict = {} + adict['startline'] = hilight_tmp['startline'] + adict['hilight_css_id'] = hilight_tmp['hilight'] + adict['endline'] = hilight_tmp['endline'] + new_hilight_dict[hilight_tmp['startline']] = adict + if not add_new_hilight_middel is None: + adict1 = {} + adict2 = {} + adict3 = {} + adict1['startline'] = new_hilight_dict[add_new_hilight_middel]['startline'] + adict1['endline'] = hilight_tmp['startline'] -1 + adict1['hilight_css_id'] = new_hilight_dict[add_new_hilight_middel]['hilight'] + adict2['startline'] = hilight_tmp['startline'] + adict2['hilight_css_id'] = hilight_tmp['hilight'] + adict2['endline'] = hilight_tmp['endline'] + adict3['startline'] = hilight_tmp['endline'] + 1 + adict3['hilight_css_id'] = new_hilight_dict[add_new_hilight_middel]['hilight'] + adict3['endline'] = new_hilight_dict[add_new_hilight_middel]['endline'] + del new_hilight_dict[add_new_hilight_middel] + new_hilight_dict[adict1['startline']] = adict1 + new_hilight_dict[adict2['startline']] = adict2 + new_hilight_dict[adict3['startline']] = adict3 + return new_hilight_dict + +def get_buildlog_info(session, settings, pkg, build_dict): + myportdb = portage.portdbapi(mysettings=settings) + init_repoman = tbc_repoman(settings, myportdb) + logfile_text_dict, max_text_lines = get_log_text_dict(settings.get("PORTAGE_LOG_FILE")) + hilight_dict = search_buildlog(session, logfile_text_dict, max_text_lines) + error_log_list = [] + qa_error_list = [] + repoman_error_list = [] + sum_build_log_list = [] + error_info_list = get_error_info_list(session) + for k, v in sorted(hilight_dict.items()): + if v['startline'] == v['endline']: + error_log_list.append(logfile_text_dict[k ]) + if v['hilight_css_id'] == "3" or v['hilight_css_id'] == "4": # qa = 3 and 4 + qa_error_list.append(logfile_text_dict[k]) + else: + i = k + while i != (v['endline'] + 1): + error_log_list.append(logfile_text_dict[i]) + if v['hilight_css_id'] == "3" or v['hilight_css_id'] == "4": # qa = 3 and 4 + qa_error_list.append(logfile_text_dict[i]) + i = i +1 + + # Run repoman check_repoman() + repoman_error_list = init_repoman.check_repoman(build_dict['cpv'], pkg.repo) + if repoman_error_list != []: + sum_build_log_list.append("1") # repoman = 1 + if qa_error_list != []: + sum_build_log_list.append("2") # qa = 2 + error_search_line = "^ \\* ERROR: " + for error_log_line in error_log_list: + if re.search(error_search_line, error_log_line): + for error_info in error_info_list: + if re.search(error_info.ErrorSearch, error_log_line): + sum_build_log_list.append(error_info.ErrorId) + build_log_dict = {} + build_log_dict['repoman_error_list'] = repoman_error_list + build_log_dict['qa_error_list'] = qa_error_list + build_log_dict['error_log_list'] = error_log_list + build_log_dict['summary_error_list'] = sum_build_log_list + build_log_dict['hilight_dict'] = hilight_dict + return build_log_dict + +def get_emerge_info_id(settings, trees, session, config_id): + args = [] + args.append("--info") + myaction, myopts, myfiles = parse_opts(args, silent=True) + status, emerge_info_list = action_info(settings, trees, myopts, myfiles) + emerge_info = "" + return "\n".join(emerge_info_list) + +def add_buildlog_main(settings, pkg, trees): + reader=get_conf_settings() + tbc_settings_dict=reader.read_tbc_settings_all() + config = tbc_settings_dict['tbc_config'] + hostname =tbc_settings_dict['hostname'] + host_config = hostname + "/" + config + Session = sessionmaker(bind=NewConnection(tbc_settings_dict)) + session = Session() + config_id = get_config_id(session, config, hostname) + if pkg.type_name == "binary": + build_dict = None + else: + build_dict = get_build_dict_db(session, config_id, settings, pkg) + if build_dict is None: + log_msg = "Package %s:%s is NOT logged." % (pkg.cpv, pkg.repo,) + add_tbc_logs(session, log_msg, "info", config_id) + session.close + return + build_log_dict = {} + build_log_dict = get_buildlog_info(session, settings, pkg, build_dict) + error_log_list = build_log_dict['error_log_list'] + build_error = "" + log_hash = hashlib.sha256() + build_error = "" + if error_log_list != []: + for log_line in error_log_list: + build_error = build_error + log_line + log_hash.update(log_line.encode('utf-8')) + build_log_dict['build_error'] = build_error + build_log_dict['log_hash'] = log_hash.hexdigest() + build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1] + log_msg = "Logfile name: %s" % (settings.get("PORTAGE_LOG_FILE"),) + add_tbc_logs(session, log_msg, "info", config_id) + build_log_dict['emerge_info'] = get_emerge_info_id(settings, trees, session, config_id) + log_id = add_new_buildlog(session, build_dict, build_log_dict) + + if log_id is None: + log_msg = "Package %s:%s is NOT logged." % (pkg.cpv, pkg.repo,) + add_tbc_logs(session, log_msg, "info", config_id) + else: + os.chmod(settings.get("PORTAGE_LOG_FILE"), 0o664) + log_msg = "Package: %s:%s is logged." % (pkg.cpv, pkg.repo,) + add_tbc_logs(session, log_msg, "info", config_id) + print("\n>>> Logging %s:%s\n" % (pkg.cpv, pkg.repo,)) + session.close + +def log_fail_queru(session, build_dict, settings): + config_id = build_dict['config_id'] + if get_fail_times(session, build_dict): + fail_querue_dict = {} + fail_querue_dict['build_job_id'] = build_dict['build_job_id'] + fail_querue_dict['fail_type'] = build_dict['type_fail'] + fail_querue_dict['fail_times'] = 1 + add_fail_times(session, fail_querue_dict) + update_buildjobs_status(session, build_dict['build_job_id'], 'Waiting', config_id) + else: + build_log_dict = {} + error_log_list = [] + qa_error_list = [] + repoman_error_list = [] + sum_build_log_list = [] + sum_build_log_list.append("2") + error_log_list.append(build_dict['type_fail']) + build_log_dict['repoman_error_list'] = repoman_error_list + build_log_dict['qa_error_list'] = qa_error_list + build_log_dict['summary_error_list'] = sum_build_log_list + if build_dict['type_fail'] == 'merge fail': + error_log_list = [] + for k, v in build_dict['failed_merge'].items(): + error_log_list.append(v['fail_msg']) + build_log_dict['error_log_list'] = error_log_list + build_error = "" + if error_log_list != []: + for log_line in error_log_list: + build_error = build_error + log_line + build_log_dict['build_error'] = build_error + summary_error = "" + if sum_build_log_list != []: + for sum_log_line in sum_build_log_list: + summary_error = summary_error + " " + sum_log_line + build_log_dict['log_hash'] = '0' + useflagsdict = {} + if build_dict['build_useflags'] == {}: + for k, v in build_dict['build_useflags'].items(): + use_id = get_use_id(session, k) + useflagsdict[use_id] = v + build_dict['build_useflags'] = useflagsdict + else: + build_dict['build_useflags'] = None + if settings.get("PORTAGE_LOG_FILE") is not None: + ConfigInfo= get_config_info(session, config_id) + host_config = ConfigInfo.Hostname +"/" + ConfigInfo.Config + build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1] + os.chmod(settings.get("PORTAGE_LOG_FILE"), 0o664) + else: + build_log_dict['logfilename'] = "" + build_log_dict['hilight_dict'] = {} + settings2, trees, tmp = load_emerge_config() + build_log_dict['emerge_info'] = get_emerge_info_id(settings2, trees, session, config_id) + log_id = add_new_buildlog(session, build_dict, build_log_dict) + del_old_build_jobs(session, build_dict['build_job_id']) diff --git a/tbc/pym/buildquerydb.py b/tbc/pym/buildquerydb.py new file mode 100644 index 0000000..7fe7f82 --- /dev/null +++ b/tbc/pym/buildquerydb.py @@ -0,0 +1,102 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import sys +import os + +# Get the options from the config file set in tbc.readconf +from tbc.readconf import get_conf_settings +reader=get_conf_settings() +tbc_settings_dict=reader.read_tbc_settings_all() +config_profile = tbc_settings_dict['tbc_config'] + +from tbc.check_setup import check_make_conf +from tbc.sync import git_pull +from tbc.package import tbc_package +import portage +import multiprocessing + +def add_cpv_query_pool(mysettings, myportdb, config_id, cp, repo): + conn =0 + init_package = tbc_package(mysettings, myportdb) + # FIXME: remove the check for tbc when in tree + if cp != "dev-python/tbc": + build_dict = {} + packageDict = {} + ebuild_id_list = [] + # split the cp to categories and package + element = cp.split('/') + categories = element[0] + package = element[1] + log_msg = "C %s:%s" % (cp, repo,) + add_tbc_logs(conn, log_msg, "info", config_id) + pkgdir = self._myportdb.getRepositoryPath(repo) + "/" + cp + config_id_list = [] + config_id_list.append(config_id) + config_cpv_listDict = init_package.config_match_ebuild(cp, config_id_list) + if config_cpv_listDict != {}: + cpv = config_cpv_listDict[config_id]['cpv'] + packageDict[cpv] = init_package.get_packageDict(pkgdir, cpv, repo) + build_dict['checksum'] = packageDict[cpv]['ebuild_version_checksum_tree'] + build_dict['package_id'] = get_package_id(conn, categories, package, repo) + build_dict['ebuild_version'] = packageDict[cpv]['ebuild_version_tree'] + ebuild_id = get_ebuild_id_db_checksum(conn, build_dict) + if ebuild_id is not None: + ebuild_id_list.append(ebuild_id) + init_package.add_new_ebuild_buildquery_db(ebuild_id_list, packageDict, config_cpv_listDict) + log_msg = "C %s:%s ... Done." % (cp, repo,) + add_tbc_logs(conn, log_msg, "info", config_id) + return + +def add_buildquery_main(config_id): + conn = 0 + config_setup = get_config(conn, config_id) + log_msg = "Adding build jobs for: %s" % (config_setup,) + add_tbc_logs(conn, log_msg, "info", config_id) + check_make_conf() + log_msg = "Check configs done" + add_tbc_logs(conn, log_msg, "info", config_profile) + # Get default config from the configs table and default_config=1 + default_config_root = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + config_setup + "/" + # Set config_root (PORTAGE_CONFIGROOT) to default_config_root + mysettings = portage.config(config_root = default_config_root) + myportdb = portage.portdbapi(mysettings=mysettings) + init_package = tbc_package(mysettings, myportdb) + log_msg = "Setting default config to: %s" % (config_setup) + add_tbc_logs(conn, log_msg, "info", config_is) + # Use all exept 2 cores when multiprocessing + pool_cores= multiprocessing.cpu_count() + if pool_cores >= 3: + use_pool_cores = pool_cores - 2 + else: + use_pool_cores = 1 + pool = multiprocessing.Pool(processes=use_pool_cores) + + repo_trees_list = myportdb.porttrees + for repo_dir in repo_trees_list: + repo = myportdb.getRepositoryName(repo_dir) + repo_dir_list = [] + repo_dir_list.append(repo_dir) + + # Get the package list from the repo + package_list_tree = myportdb.cp_all(trees=repo_dir_list) + for cp in sorted(package_list_tree): + pool.apply_async(add_cpv_query_pool, (mysettings, myportdb, config_id, cp, repo,)) + pool.close() + pool.join() + log_msg = "Adding build jobs for: %s ... Done." % (config_setup,) + add_tbc_logs(conn, log_msg, "info", config_profile) + return True + +def del_buildquery_main(config_id): + conn=0 + config_setup = get_config(conn, config_id) + log_msg = "Removeing build jobs for: %s" % (config_setup,) + add_tbc_logs(conn, log_msg, "info", config_id) + build_job_id_list = get_build_jobs_id_list_config(conn, config_id) + if build_job_id_list is not None: + for build_job_id in build_job_id_list: + del_old_build_jobs(conn, build_job_id) + log_msg = "Removeing build jobs for: %s ... Done." % (config_setup,) + add_tbc_logs(conn, log_msg, "info", config_id) + return True diff --git a/tbc/pym/check_setup.py b/tbc/pym/check_setup.py new file mode 100644 index 0000000..46d4391 --- /dev/null +++ b/tbc/pym/check_setup.py @@ -0,0 +1,74 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +import portage +import os +import errno + +from portage.exception import DigestException, FileNotFound, ParseError, PermissionDenied +from tbc.text import get_file_text +from tbc.sqlquerys import get_config_all_info, add_tbc_logs, get_configmetadata_info, get_setup_info + +def check_make_conf(session, config_id, tbc_settings_dict): + log_msg = "Checking configs for changes and errors" + add_tbc_logs(session, log_msg, "info", config_id) + configsDict = {} + for ConfigInfo in get_config_all_info(session): + attDict={} + # Set the config dir + SetupInfo = get_setup_info(session, ConfigInfo.ConfigId) + check_config_dir = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + ConfigInfo.Hostname +"/" + SetupInfo.Setup + "/" + make_conf_file = check_config_dir + "etc/portage/make.conf" + ConfigsMetaDataInfo = get_configmetadata_info(session, ConfigInfo.ConfigId) + # Check if we can take a checksum on it. + # Check if we have some error in the file. (portage.util.getconfig) + # Check if we envorment error with the config. (settings.validate) + try: + make_conf_checksum_tree = portage.checksum.sha256hash(make_conf_file)[0] + portage.util.getconfig(make_conf_file, tolerant=0, allow_sourcing=True, expand=True) + mysettings = portage.config(config_root = check_config_dir) + mysettings.validate() + # With errors we update the db on the config and disable the config + except ParseError as e: + ConfigsMetaDataInfo.ConfigErrorText = str(e) + ConfigsMetaDataInfo.Active = False + log_msg = "%s FAIL!" % (ConfigInfo.Hostname,) + add_tbc_logs(session, log_msg, "info", config_id) + session.commit() + else: + ConfigsMetaDataInfo.Active = True + log_msg = "%s PASS" % (ConfigInfo.Hostname,) + add_tbc_logs(session, log_msg, "info", config_id) + session.commit() + if make_conf_checksum_tree != ConfigsMetaDataInfo.Checksum: + ConfigsMetaDataInfo.MakeConfText = get_file_text(make_conf_file) + ConfigsMetaDataInfo.Checksum = make_conf_checksum_tree + session.commit() + log_msg = "Checking configs for changes and errors ... Done" + add_tbc_logs(session, log_msg, "info", config_id) + +def check_make_conf_guest(session, config_id): + make_conf_file = "/etc/portage/make.conf" + # Check if we can open the file and close it + # Check if we have some error in the file (portage.util.getconfig) + # Check if we envorment error with the config (settings.validate) + try: + make_conf_checksum_tree = portage.checksum.sha256hash(make_conf_file)[0] + portage.util.getconfig(make_conf_file, tolerant=0, allow_sourcing=True, expand=True) + mysettings = portage.config(config_root = "/") + mysettings.validate() + # With errors we return false + except Exception as e: + return False + ConfigsMetaDataInfo = get_configmetadata_info(session, config_id) + print('make_conf_checksum_tree', make_conf_checksum_tree) + print('make_conf_checksum_db', ConfigsMetaDataInfo.Checksum) + if make_conf_checksum_tree != ConfigsMetaDataInfo.Checksum: + return False + return True + +def check_configure_guest(session, config_id): + pass_make_conf = check_make_conf_guest(session, config_id) + print(pass_make_conf) + return pass_make_conf \ No newline at end of file diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py new file mode 100644 index 0000000..c215456 --- /dev/null +++ b/tbc/pym/db_mapping.py @@ -0,0 +1,260 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import datetime +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, Integer, String, Boolean, DateTime, Enum, Text, ForeignKey +from sqlalchemy.orm import relationship, backref + +Base = declarative_base() + +class Keywords(Base): + KeywordId = Column('keyword_id', Integer, primary_key=True) + Keyword = Column('keyword', String) + __tablename__ = 'keywords' + +class Setups(Base): + SetupId = Column('setup_id', Integer, primary_key=True) + Setup = Column('setup', String(100)) + __tablename__ = 'setups' + +class Configs(Base): + ConfigId = Column('config_id', Integer, primary_key=True) + Hostname = Column('hostname', String(150)) + SetupId = Column('setup_id', Integer, ForeignKey('setups.setup_id')) + Host = Column('default_config', Boolean, default=False) + __tablename__ = 'configs' + +class Logs(Base): + LogId = Column('log_id', Integer, primary_key=True) + ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id')) + LogType = Column('log_type', Enum('info','error','debug','qa','repoman')) + Msg = Column('msg', Text) + TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) + __tablename__ = 'logs' + +class Jobs(Base): + JobId = Column('job_id', Integer, primary_key=True) + JobType = Column('job_type', Enum('updatedb', 'esync', 'gsync')) + Status = Column('status', Enum('Runing', 'Done', 'Waiting')) + User = Column('user', String(50)) + ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id')) + RunConfigId = Column('run_config_id', Integer, ForeignKey('configs.config_id')) + TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) + __tablename__ = 'jobs' + +class ConfigsMetaData(Base): + Id = Column('id', Integer, primary_key=True) + ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id')) + Profile = Column('profile', String(150)) + KeywordId = Column('keyword_id', Integer, ForeignKey('keywords.keyword_id')) + MakeConfText = Column('make_conf_text', Text) + Checksum = Column('checksum', String(100)) + ConfigSync = Column('configsync', Boolean, default=False) + Active = Column('active', Boolean, default=False) + ConfigErrorText = Column('config_error_text', Text) + Updateing = Column('updateing', Boolean, default=False) + Status = Column('status', Enum('Stopped', 'Runing', 'Waiting')) + Auto = Column('auto', Boolean, default=False) + GitWww = Column('git_www', String(200)) + TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) + __tablename__ = 'configs_metadata' + +class Categories(Base): + CategoryId = Column('category_id', Integer, primary_key=True) + Category = Column('category', String(150)) + Active = Column('active', Boolean, default=True) + TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) + __tablename__ = 'categories' + +class Repos(Base): + RepoId = Column('repo_id', Integer, primary_key=True) + Repo = Column('repo', String(100)) + __tablename__ = 'repos' + +class Packages(Base): + PackageId = Column('package_id', Integer, primary_key=True) + CategoryId = Column('category_id', Integer, ForeignKey('categories.category_id')) + Package = Column('package',String(150)) + RepoId = Column('repo_id', Integer, ForeignKey('repos.repo_id')) + Checksum = Column('checksum', String(100)) + Active = Column('active', Boolean, default=False) + TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) + __tablename__ = 'packages' + +class Emails(Base): + EmailId = Column('email_id', Integer, primary_key=True) + Email = Column('email', String(150)) + __tablename__ = 'emails' + +class PackagesEmails(Base): + Id = Column('id', Integer, primary_key=True) + PackageId = Column('package_id', Integer, ForeignKey('packages.package_id')) + EmailId = Column('email_id', Integer, ForeignKey('emails.email_id')) + __tablename__ = 'packages_emails' + +class PackagesMetadata(Base): + Id = Column('id', Integer, primary_key=True) + PackageId = Column('package_id', Integer, ForeignKey('packages.package_id')) + Checksum = Column('checksum', String(100)) + __tablename__ = 'packages_metadata' + +class Ebuilds(Base): + EbuildId = Column('ebuild_id', Integer, primary_key=True) + PackageId = Column('package_id', Integer, ForeignKey('packages.package_id')) + Version = Column('version', String(150)) + Checksum = Column('checksum', String(100)) + Active = Column('active', Boolean, default=False) + TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) + __tablename__ = 'ebuilds' + +class EmergeOptions(Base): + EmergeOptionId = Column('eoption_id', Integer, primary_key=True) + EOption = Column('eoption', String(45)) + __tablename__ = 'emerge_options' + +class ConfigsEmergeOptions(Base): + ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'), primary_key=True) + EOptionId = Column('eoption_id', Integer, ForeignKey('emerge_options.eoption_id')) + __tablename__ = 'configs_emerge_options' + +class BuildJobs(Base): + BuildJobId = Column('build_job_id', Integer, primary_key=True) + EbuildId = Column('ebuild_id', Integer, ForeignKey('ebuilds.ebuild_id')) + SetupId = Column('setup_id', Integer, ForeignKey('setups.setup_id')) + ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id')) + Status = Column('status', Enum('Waiting','Building','Looked',)) + BuildNow = Column('build_now', Boolean, default=False) + RemoveBin = Column('removebin', Boolean ,default=False) + TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) + __tablename__ = 'build_jobs' + +class BuildJobsEmergeOptions(Base): + Id = Column('id', Integer, primary_key=True) + BuildJobId = Column('build_job_id', Integer, ForeignKey('build_jobs.build_job_id')) + EOption = Column('eoption_id', Integer, ForeignKey('emerge_options.eoption_id')) + __tablename__ = 'build_jobs_emerge_options' + +class BuildJobsRedo(Base): + Id = Column('id', Integer, primary_key=True) + BuildJobId = Column('build_job_id', Integer, ForeignKey('build_jobs.build_job_id')) + FailTimes = Column('fail_times', Integer) + FailType = Column('fail_type', String(50)) + TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) + __tablename__ = 'build_jobs_redo' + +class Uses(Base): + UseId = Column('use_id', Integer, primary_key=True) + Flag = Column('flag', String(150)) + __tablename__ = 'uses' + +class BuildJobsUse(Base): + Id = Column('id', Integer, primary_key=True) + BuildJobId = Column('build_job_id', Integer, ForeignKey('build_jobs.build_job_id')) + UseId = Column('use_id', Integer, ForeignKey('uses.use_id')) + Status = Column('status', Boolean, default=False) + __tablename__ = 'build_jobs_use' + +class HiLightCss(Base): + HiLightCssId = Column('hilight_css_id', Integer, primary_key=True) + HiLightCssName = Column('hilight_css_name', String(30)) + HiLightCssCollor = Column('hilight_css_collor', String(30)) + __tablename__ = 'hilight_css' + +class HiLight(Base): + HiLightId = Column('hilight_id', Integer, primary_key=True) + HiLightSearch = Column('hilight_search', String(50)) + HiLightSearchEnd = Column('hilight_search_end', String(50)) + HiLightSearchPattern = Column('hilight_search_pattern', String(50)) + HiLightCssId = Column('hilight_css_id', Integer, ForeignKey('hilight_css.hilight_css_id')) + HiLightStart = Column('hilight_start', Integer) + HiLightEnd = Column('hilight_end', Integer) + __tablename__ = 'hilight' + +class BuildLogs(Base): + BuildLogId = Column('build_log_id', Integer, primary_key=True) + EbuildId = Column('ebuild_id', Integer, ForeignKey('ebuilds.ebuild_id')) + Fail = Column('fail', Boolean, default=False) + SummeryText = Column('summery_text', Text) + LogHash = Column('log_hash', String(100)) + BugId = Column('bug_id', Integer, default=0) + TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) + __tablename__ = 'build_logs' + +class EmergeInfo(Base): + EInfoId = Column('einfo_id', Integer, primary_key=True) + EmergeInfoText = Column('emerge_info_text', Text) + __tablename__ = 'emerge_info' + +class BuildLogsConfig(Base): + LogId = Column('log_id', Integer, primary_key=True) + BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id')) + ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id')) + EInfoId = Column('einfo_id', Integer, ForeignKey('emerge_info.einfo_id')) + LogName = Column('logname', String(450)) + TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow) + __tablename__ = 'build_logs_config' + +class BuildLogsHiLight(Base): + BuildLogHiLightId = Column('id', Integer, primary_key=True) + LogId = Column('log_id', Integer, ForeignKey('build_logs_config.log_id')) + StartLine = Column('start_line', Integer) + EndLine = Column('end_line', Integer) + HiLightCssId = Column('hilight_css_id', Integer, ForeignKey('hilight_css.hilight_css_id')) + __tablename__ = 'build_logs_hilight' + +class BuildLogsEmergeOptions(Base): + Id = Column('id', Integer, primary_key=True) + BuildLogId = Column('build_log_id', Integer, ForeignKey('Build_logs.Build_log_id')) + EmergeOptionId = Column('eoption_id', Integer, ForeignKey('emerge_options.eoption_id')) + __tablename__ = 'build_logs_emerge_options' + +class BuildLogsUse(Base): + Id = Column('id', Integer, primary_key=True) + BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id')) + UseId = Column('use_id', Integer, ForeignKey('uses.use_id')) + Status = Column('status', Boolean, default=False) + __tablename__ = 'build_logs_use' + +class ErrorsInfo(Base): + ErrorId = Column('error_id', Integer, primary_key=True) + ErrorName = Column('error_name', String) + ErrorSearch = Column('error_search', String) + __tablename__ = 'errors_info' + +class BuildLogsErrors(Base): + BuildLogErrorId = Column('id', Integer, primary_key=True) + BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id')) + ErrorId = Column('error_id', Integer, ForeignKey('errors_info.error_id')) + __tablename__ = 'build_logs_errors' + +class Restrictions(Base): + RestrictionId = Column('restriction_id', Integer, primary_key=True) + Restriction = Column('restriction', String(150)) + __tablename__ = 'restrictions' + +class EbuildsRestrictions(Base): + Id = Column('id', Integer, primary_key=True) + EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id')) + RestrictionId = Column('restriction_id', ForeignKey('restrictions.restriction_id')) + __tablename__ = 'ebuilds_restrictions' + +class EbuildsIUse(Base): + Id = Column('id', Integer, primary_key=True) + EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id')) + UseId = Column('use_id', ForeignKey('uses.use_id')) + Status = Column('status', Boolean, default=False) + __tablename__= 'ebuilds_iuse' + +class EbuildsKeywords(Base): + Id = Column('id', Integer, primary_key=True) + EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id')) + KeywordId = Column('keyword_id', ForeignKey('keywords.keyword_id')) + Status = Column('status', Enum('Stable','Unstable','Negative')) + __tablename__ = 'ebuilds_keywords' + +class EbuildsMetadata(Base): + Id = Column('id', Integer, primary_key=True) + EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id')) + Revision = Column('revision', String(30)) + __tablename__ = 'ebuilds_metadata' diff --git a/tbc/pym/depclean.py b/tbc/pym/depclean.py new file mode 100644 index 0000000..3154ac5 --- /dev/null +++ b/tbc/pym/depclean.py @@ -0,0 +1,53 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +import portage +from portage._sets.base import InternalPackageSet +from _emerge.main import parse_opts +from tbc.actions import load_emerge_config, action_depclean, calc_depclean + +def do_depclean(): + mysettings, mytrees, mtimedb = load_emerge_config() + myroot = mysettings["ROOT"] + root_config = mytrees[myroot]["root_config"] + psets = root_config.setconfig.psets + args_set = InternalPackageSet(allow_repo=True) + spinner=None + scheduler=None + tmpcmdline = [] + tmpcmdline.append("--depclean") + tmpcmdline.append("--pretend") + print("depclean",tmpcmdline) + myaction, myopts, myfiles = parse_opts(tmpcmdline, silent=False) + if myfiles: + args_set.update(myfiles) + matched_packages = False + for x in args_set: + if vardb.match(x): + matched_packages = True + if not matched_packages: + return 0 + + rval, cleanlist, ordered, req_pkg_count, unresolvable = calc_depclean(mysettings, mytrees, mtimedb["ldpath"], myopts, myaction, args_set, spinner) + print('rval, cleanlist, ordered, req_pkg_count, unresolvable', rval, cleanlist, ordered, req_pkg_count, unresolvable) + if unresolvable != []: + return True + if cleanlist != []: + conflict_package_list = [] + for depclean_cpv in cleanlist: + if portage.versions.cpv_getkey(depclean_cpv) in list(psets["system"]): + conflict_package_list.append(depclean_cpv) + if portage.versions.cpv_getkey(depclean_cpv) in list(psets['selected']): + conflict_package_list.append(depclean_cpv) + print('conflict_package_list', conflict_package_list) + if conflict_package_list == []: + tmpcmdline = [] + tmpcmdline.append("--depclean") + myaction, myopts, myfiles = parse_opts(tmpcmdline, silent=False) + rval = action_depclean(mysettings, mytrees, mtimedb["ldpath"], myopts, myaction, myfiles, spinner, scheduler=None) + return True + else: + print("conflicting packages: %s", conflict_package_list) + return True + return True diff --git a/tbc/pym/flags.py b/tbc/pym/flags.py new file mode 100644 index 0000000..eea48f3 --- /dev/null +++ b/tbc/pym/flags.py @@ -0,0 +1,231 @@ +#!/usr/bin/python +# +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# Origin flags.py from portage public api repo +from __future__ import print_function +from _emerge.main import parse_opts +from _emerge.depgraph import backtrack_depgraph, depgraph, resume_depgraph +from _emerge.create_depgraph_params import create_depgraph_params +from _emerge.actions import load_emerge_config +import portage +import os + +class tbc_use_flags(object): + + def __init__(self, mysettings, myportdb, cpv): + self._mysettings = mysettings + self._myportdb = myportdb + self._cpv = cpv + + def get_iuse(self): + """Gets the current IUSE flags from the tree + To be used when a gentoolkit package object is not needed + @type: cpv: string + @param cpv: cat/pkg-ver + @rtype list + @returns [] or the list of IUSE flags + """ + return self._myportdb.aux_get(self._cpv, ["IUSE"])[0].split() + + def reduce_flag(self, flag): + """Absolute value function for a USE flag + @type flag: string + @param flag: the use flag to absolute. + @rtype: string + @return absolute USE flag + """ + if flag[0] in ["+","-"]: + return flag[1:] + else: + return flag + + def reduce_flags(self, the_list): + """Absolute value function for a USE flag list + @type the_list: list + @param the_list: the use flags to absolute. + @rtype: list + @return absolute USE flags + """ + r=[] + for member in the_list: + r.append(self.reduce_flag(member)) + return r + + def filter_flags(self, use, use_expand_hidden, usemasked, useforced): + """Filter function to remove hidden or otherwise not normally + visible USE flags from a list. + @type use: list + @param use: the USE flag list to be filtered. + @type use_expand_hidden: list + @param use_expand_hidden: list of flags hidden. + @type usemasked: list + @param usemasked: list of masked USE flags. + @type useforced: list + @param useforced: the forced USE flags. + @rtype: list + @return the filtered USE flags. + """ + # clean out some environment flags, since they will most probably + # be confusing for the user + for f in use_expand_hidden: + f=f.lower() + "_" + for x in use: + if f in x: + use.remove(x) + # clean out any arch's + archlist = self._mysettings["PORTAGE_ARCHLIST"].split() + for a in use[:]: + if a in archlist: + use.remove(a) + # clean out any abi_ flag + for a in use[:]: + if a.startswith("abi_"): + use.remove(a) + # clean out any python_ flag + for a in use[:]: + if a.startswith("python_"): + use.remove(a) + + # dbl check if any from usemasked or useforced are still there + masked = usemasked + useforced + for a in use[:]: + if a in masked: + use.remove(a) + return use + + def get_all_cpv_use(self): + """Uses portage to determine final USE flags and settings for an emerge + @type cpv: string + @param cpv: eg cat/pkg-ver + @rtype: lists + @return use, use_expand_hidden, usemask, useforce + """ + use = None + self._mysettings.unlock() + try: + self._mysettings.setcpv(self._cpv, use_cache=None, mydb=self._myportdb) + use = self._mysettings['PORTAGE_USE'].split() + use_expand_hidden = self._mysettings["USE_EXPAND_HIDDEN"].split() + usemask = list(self._mysettings.usemask) + useforce = list(self._mysettings.useforce) + except KeyError: + self._mysettings.reset() + self._mysettings.lock() + return [], [], [], [] + # reset cpv filter + self._mysettings.reset() + self._mysettings.lock() + print(usemask) + return use, use_expand_hidden, usemask, useforce + + def get_all_cpv_use_looked(self): + """Uses portage to determine final USE flags and settings for an emerge + @type cpv: string + @param cpv: eg cat/pkg-ver + @rtype: lists + @return use, use_expand_hidden, usemask, useforce + """ + # use = self._mysettings['PORTAGE_USE'].split() + use = os.environ['USE'].split() + use_expand_hidden = self._mysettings["USE_EXPAND_HIDDEN"].split() + usemask = list(self._mysettings.usemask) + useforce = list(self._mysettings.useforce) + return use, use_expand_hidden, usemask, useforce + + def get_all_cpv_use_pkg(self, pkg, settings): + """Uses portage to determine final USE flags and settings for an emerge + @type cpv: string + @param cpv: eg cat/pkg-ver + @rtype: lists + @return use, use_expand_hidden, usemask, useforce + """ + # use = self._mysettings['PORTAGE_USE'].split() + use_list = list(pkg.use.enabled) + use_expand_hidden = settings["USE_EXPAND_HIDDEN"].split() + usemask = list(settings.usemask) + useforced = list(settings.useforce) + return use_list, use_expand_hidden, usemask, useforced + + def get_flags(self): + """Retrieves all information needed to filter out hidden, masked, etc. + USE flags for a given package. + + @type cpv: string + @param cpv: eg. cat/pkg-ver + @type final_setting: boolean + @param final_setting: used to also determine the final + enviroment USE flag settings and return them as well. + @rtype: list or list, list + @return IUSE or IUSE, final_flags + """ + final_use, use_expand_hidden, usemasked, useforced = self.get_all_cpv_use() + iuse_flags = self.filter_flags(self.get_iuse(), use_expand_hidden, usemasked, useforced) + #flags = filter_flags(use_flags, use_expand_hidden, usemasked, useforced) + final_flags = self.filter_flags(final_use, use_expand_hidden, usemasked, useforced) + return iuse_flags, final_flags + + def get_flags_looked(self): + """Retrieves all information needed to filter out hidden, masked, etc. + USE flags for a given package. + + @type cpv: string + @param cpv: eg. cat/pkg-ver + @type final_setting: boolean + @param final_setting: used to also determine the final + enviroment USE flag settings and return them as well. + @rtype: list or list, list + @return IUSE or IUSE, final_flags + """ + final_use, use_expand_hidden, usemasked, useforced = self.get_all_cpv_use_looked() + iuse_flags = self.filter_flags(self.get_iuse(), use_expand_hidden, usemasked, useforced) + #flags = filter_flags(use_flags, use_expand_hidden, usemasked, useforced) + final_flags = self.filter_flags(final_use, use_expand_hidden, usemasked, useforced) + return iuse_flags, final_flags + + def get_flags_pkg(self, pkg, settings): + """Retrieves all information needed to filter out hidden, masked, etc. + USE flags for a given package. + @type cpv: string + @param cpv: eg. cat/pkg-ver + @type final_setting: boolean + @param final_setting: used to also determine the final + enviroment USE flag settings and return them as well. + @rtype: list or list, list + @return IUSE or IUSE, final_flags + """ + final_use, use_expand_hidden, usemasked, useforced = self.get_all_cpv_use_pkg(pkg, settings) + iuse_flags = self.filter_flags(list(pkg.iuse.all), use_expand_hidden, usemasked, useforced) + #flags = filter_flags(use_flags, use_expand_hidden, usemasked, useforced) + final_flags = self.filter_flags(final_use, use_expand_hidden, usemasked, useforced) + return iuse_flags, final_flags + + def comper_useflags(self, build_dict): + iuse_flags, use_enable = self.get_flags() + iuse = [] + build_use_flags_dict = build_dict['build_useflags'] + build_use_flags_list = [] + if use_enable == []: + if build_use_flags_dict is None: + return None + for iuse_line in iuse_flags: + iuse.append(self.reduce_flag(iuse_line)) + iuse_flags_list = list(set(iuse)) + use_disable = list(set(iuse_flags_list).difference(set(use_enable))) + use_flagsDict = {} + for x in use_enable: + use_flagsDict[x] = True + for x in use_disable: + use_flagsDict[x] = False + print("use_flagsDict", use_flagsDict) + for k, v in use_flagsDict.items(): + if build_use_flags_dict[k] != v: + if build_use_flags_dict[k]: + build_use_flags_list.append(k) + else: + build_use_flags_list.append("-" + k) + if build_use_flags_list == []: + build_use_flags_list = None + print(build_use_flags_list) + return build_use_flags_list diff --git a/tbc/pym/jobs.py b/tbc/pym/jobs.py new file mode 100644 index 0000000..840932e --- /dev/null +++ b/tbc/pym/jobs.py @@ -0,0 +1,86 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +from tbc.sync import sync_tree +#from tbc.buildquerydb import add_buildquery_main, del_buildquery_main +from tbc.updatedb import update_db_main +from tbc.sqlquerys import get_config_id, add_tbc_logs, get_jobs, update_job_list + +def jobs_main(session, config_id): + JobsInfo = get_jobs(session, config_id) + if JobsInfo is None: + return + for JobInfo in JobsInfo: + job = JobInfo.JobType + run_config_id = JobInfo.RunConfigId + job_id = JobInfo.JobId + log_msg = "Job: %s Type: %s" % (job_id, job,) + add_tbc_logs(session, log_msg, "info", config_id) + if job == "addbuildquery": + update_job_list(session, "Runing", job_id) + log_msg = "Job %s is runing." % (job_id,) + add_tbc_logs(session, log_msg, "info", config_id) + #result = add_buildquery_main(run_config_id) + #if result is True: + # update_job_list(session, "Done", job_id) + # log_msg = "Job %s is done.." % (job_id,) + # add_tbc_logs(session, log_msg, "info", config_id) + #else: + # update_job_list(session, "Fail", job_id) + # log_msg = "Job %s did fail." % (job_id,) + # add_tbc_logs(session, log_msg, "info", config_id) + elif job == "delbuildquery": + update_job_list(session, "Runing", job_id) + log_msg = "Job %s is runing." % (job_id,) + add_tbc_logs(session, log_msg, "info", config_id) + #result = del_buildquery_main(config_id) + #if result is True: + # update_job_list(session, "Done", job_id) + # log_msg = "Job %s is done.." % (job_id,) + # add_tbc_logs(session, log_msg, "info", config_id) + #else: + # update_job_list(session, "Fail", job_id) + # log_msg = "Job %s did fail." % (job_id,) + # add_tbc_logs(session, log_msg, "info", config_id) + elif job == "gsync": + update_job_list(session, "Runing", job_id) + log_msg = "Job %s is runing." % (job_id,) + add_tbc_logs(session, log_msg, "info", config_id) + result = True + # result = git_pull(session) + if result: + update_job_list(session, "Done", job_id) + log_msg = "Job %s is done.." % (job_id,) + add_tbc_logs(session, log_msg, "info", config_id) + else: + update_job_list(session, "Fail", job_id) + log_msg = "Job %s did fail." % (job_id,) + add_tbc_logs(session, log_msg, "info", config_id) + elif job == "esync": + update_job_list(session, "Runing", job_id) + log_msg = "Job %s is runing." % (job_id,) + add_tbc_logs(session, log_msg, "info", config_id) + result = sync_tree(session) + if result: + update_job_list(session, "Done", job_id) + log_msg = "Job %s is done.." % (job_id,) + add_tbc_logs(session, log_msg, "info", config_id) + else: + update_job_list(session, "Fail", job_id) + log_msg = "Job %s did fail." % (job_id,) + add_tbc_logs(session, log_msg, "info", config_id) + elif job == "updatedb": + update_job_list(session, "Runing", job_id) + log_msg = "Job %s is runing." % (job_id,) + add_tbc_logs(session, log_msg, "info", config_id) + result = update_db_main(session, config_id) + if result: + update_job_list(session, "Done", job_id) + log_msg = "Job %s is done.." % (job_id,) + add_tbc_logs(session, log_msg, "info", config_id) + else: + update_job_list(session, "Fail", job_id) + log_msg = "Job %s did fail." % (job_id,) + add_tbc_logs(session, log_msg, "info", config_id) + return diff --git a/tbc/pym/old_cpv.py b/tbc/pym/old_cpv.py new file mode 100644 index 0000000..dbeba56 --- /dev/null +++ b/tbc/pym/old_cpv.py @@ -0,0 +1,97 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +import logging +from tbc.readconf import get_conf_settings +reader=get_conf_settings() +tbc_settings_dict=reader.read_tbc_settings_all() +# make a CM +from tbc.ConnectionManager import connectionManager +CM=connectionManager(tbc_settings_dict) +#selectively import the pgsql/mysql querys +if CM.getName()=='pgsql': + from tbc.pgsql_querys import * + +class tbc_old_cpv(object): + + def __init__(self, myportdb, mysettings): + self._mysettings = mysettings + self._myportdb = myportdb + + def mark_old_ebuild_db(self, package_id): + conn=CM.getConnection() + # Get the ebuild list for cp + cp, repo = get_cp_repo_from_package_id(conn, package_id) + mytree = [] + mytree.append(self._myportdb.getRepositoryPath(repo)) + ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree) + # Get ebuild list on categories, package in the db + ebuild_list_db = cp_list_db(conn, package_id) + # Check if don't have the ebuild in the tree + # Add it to the no active list + old_ebuild_list = [] + for ebuild_line in ebuild_list_db: + cpv_db = cp + "-" + ebuild_line[0] + if not cpv_db in ebuild_list_tree: + old_ebuild_list.append(ebuild_line) + # Set no active on ebuilds in the db that no longer in tree + if old_ebuild_list != []: + for old_ebuild in old_ebuild_list: + logging.info("O %s/%s-%s", categories, package, old_ebuild[0]) + add_old_ebuild(conn,package_id, old_ebuild_list) + # Check if we have older no activ ebuilds then 60 days + ebuild_old_list_db = cp_list_old_db(conn,package_id) + # Delete older ebuilds in the db + if ebuild_old_list_db != []: + for del_ebuild_old in ebuild_old_list_db: + logging.info("D %s/%s-%s", categories, package, del_ebuild_old[1]) + del_old_ebuild(conn,ebuild_old_list_db) + CM.putConnection(conn) + + def mark_old_package_db(self, package_id_list_tree): + conn=CM.getConnection() + # Get categories/package list from db + package_list_db = cp_all_db(conn) + old_package_id_list = [] + # Check if don't have the categories/package in the tree + # Add it to the no active list + for package_line in package_list_db: + if not package_line in package_id_list_tree: + old_package_id_list.append(package_line) + # Set no active on categories/package and ebuilds in the db that no longer in tree + if old_package_id_list != []: + mark_old_list = add_old_package(conn,old_package_id_list) + if mark_old_list != []: + for x in mark_old_list: + element = get_cp_from_package_id(conn,x) + logging.info("O %s", element[0]) + # Check if we have older no activ categories/package then 60 days + del_package_id_old_list = cp_all_old_db(conn,old_package_id_list) + # Delete older categories/package and ebuilds in the db + if del_package_id_old_list != []: + for i in del_package_id_old_list: + element = get_cp_from_package_id(conn,i) + logging.info("D %s", element) + del_old_package(conn,del_package_id_old_list) + CM.putConnection(conn) + + def mark_old_categories_db(self): + conn=CM.getConnection() + # Get categories list from the tree and db + categories_list_tree = self._mysettings.categories + categories_list_db =get_categories_db(conn) + categories_old_list = [] + # Check if don't have the categories in the tree + # Add it to the no active list + for categories_line in categories_list_db: + if not categories_line[0] in categories_list_tree: + old_c = get_old_categories(conn,categories_line[0]) + if old_c is not None: + categories_old_list.append(categories_line) + # Delete older categories in the db + if categories_old_list != []: + for real_old_categories in categories_old_list: + del_old_categories(conn,real_old_categoriess) + logging.info("D %s", real_old_categories) + CM.putConnection(conn) \ No newline at end of file diff --git a/tbc/pym/package.py b/tbc/pym/package.py new file mode 100644 index 0000000..af1bff5 --- /dev/null +++ b/tbc/pym/package.py @@ -0,0 +1,355 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +import portage +from portage.xml.metadata import MetaDataXML +from tbc.flags import tbc_use_flags +from tbc.manifest import tbc_manifest +from tbc.text import get_ebuild_cvs_revision +from tbc.flags import tbc_use_flags +from tbc.sqlquerys import add_tbc_logs, get_package_info, get_config_info, \ + add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \ + get_package_metadata_sql, update_package_metadata, update_manifest_sql, \ + get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \ + get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info +from tbc.readconf import get_conf_settings + +class tbc_package(object): + + def __init__(self, session, mysettings, myportdb, config_id, tbc_settings_dict): + self._session = session + self._mysettings = mysettings + self._myportdb = myportdb + self._config_id = config_id + self._tbc_settings_dict = tbc_settings_dict + + def change_config(self, host_config): + # Change config_root config_setup = table config + my_new_setup = "/var/cache/tbc/" + self._tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/" + mysettings_setup = portage.config(config_root = my_new_setup) + return mysettings_setup + + def config_match_ebuild(self, cp, config_list): + config_cpv_listDict ={} + if config_list == []: + return config_cpv_listDict + for config_id in config_list: + ConfigInfo = get_config_info(self._session, config_id) + ConfigsMetaData = get_configmetadata_info(self._session, config_id) + if ConfigsMetaData.Auto and ConfigsMetaData.Active and ConfigsMetaData.Status != 'Stopped': + SetupInfo = get_setup_info(self._session, config_id) + mysettings_setup = self.change_config(ConfigInfo.Hostname + "/" + SetupInfo.Setup) + myportdb_setup = portage.portdbapi(mysettings=mysettings_setup) + + # Get the latest cpv from portage with the config that we can build + build_cpv = myportdb_setup.xmatch('bestmatch-visible', cp) + + # Check if could get cpv from portage and add it to the config_cpv_listDict. + if build_cpv != "" and not ConfigInfo.SetupId in config_cpv_listDict: + + # Get the iuse and use flags for that config/setup and cpv + init_useflags = tbc_use_flags(mysettings_setup, myportdb_setup, build_cpv) + iuse_flags_list, final_use_list = init_useflags.get_flags() + iuse_flags_list2 = [] + for iuse_line in iuse_flags_list: + iuse_flags_list2.append( init_useflags.reduce_flag(iuse_line)) + + # Dict the needed info + attDict = {} + attDict['cpv'] = build_cpv + attDict['useflags'] = final_use_list + attDict['iuse'] = iuse_flags_list2 + config_cpv_listDict[ConfigInfo.SetupId] = attDict + + # Clean some cache + myportdb_setup.close_caches() + portage.portdbapi.portdbapi_instances.remove(myportdb_setup) + return config_cpv_listDict + + def get_ebuild_metadata(self, cpv, repo): + # Get the auxdbkeys infos for the ebuild + try: + ebuild_auxdb_list = self._myportdb.aux_get(cpv, portage.auxdbkeys, myrepo=repo) + except: + ebuild_auxdb_list = None + else: + for i in range(len(ebuild_auxdb_list)): + if ebuild_auxdb_list[i] == '': + ebuild_auxdb_list[i] = '' + return ebuild_auxdb_list + + def get_packageDict(self, pkgdir, cpv, repo): + + #Get categories, package and version from cpv + ebuild_version_tree = portage.versions.cpv_getversion(cpv) + element = portage.versions.cpv_getkey(cpv).split('/') + categories = element[0] + package = element[1] + + # Make a checksum of the ebuild + try: + ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")[0] + except: + ebuild_version_checksum_tree = "0" + log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + log_msg = "C %s:%s ... Fail." % (cpv, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + ebuild_version_cvs_revision_tree = '0' + else: + ebuild_version_cvs_revision_tree = get_ebuild_cvs_revision(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild") + + # Get the ebuild metadata + ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo) + # if there some error to get the metadata we add rubish to the + # ebuild_version_metadata_tree and set ebuild_version_checksum_tree to 0 + # so it can be updated next time we update the db + if ebuild_version_metadata_tree is None: + log_msg = " QA: %s have broken metadata on repo %s" % (cpv, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + ebuild_version_metadata_tree = ['','','','','','','','','','','','','','','','','','','','','','','','',''] + ebuild_version_checksum_tree = '0' + + # add the ebuild info to the dict packages + PackageInfo = get_package_info(self._session, categories, package, repo) + attDict = {} + attDict['package_id'] = PackageInfo.PackageId + attDict['repo'] = repo + attDict['ebuild_version'] = ebuild_version_tree + attDict['checksum']= ebuild_version_checksum_tree + attDict['ebuild_version_metadata_tree'] = ebuild_version_metadata_tree + #attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0] + attDict['ebuild_version_revision_tree'] = ebuild_version_cvs_revision_tree + return attDict + + def add_new_build_job_db(self, ebuild_id_list, packageDict, config_cpv_listDict): + # Get the needed info from packageDict and config_cpv_listDict and put that in buildqueue + # Only add it if ebuild_version in packageDict and config_cpv_listDict match + if config_cpv_listDict is not None: + # Unpack config_cpv_listDict + for setup_id, v in config_cpv_listDict.items(): + build_cpv = v['cpv'] + iuse_flags_list = list(set(v['iuse'])) + use_enable= v['useflags'] + use_disable = list(set(iuse_flags_list).difference(set(use_enable))) + # Make a dict with enable and disable use flags for ebuildqueuedwithuses + use_flagsDict = {} + for x in use_enable: + use_flagsDict[x] = True + for x in use_disable: + use_flagsDict[x] = False + # Unpack packageDict + i = 0 + for k, v in packageDict.items(): + ebuild_id = ebuild_id_list[i] + + # Comper and add the cpv to buildqueue + if build_cpv == k: + add_new_build_job(self._session, ebuild_id, setup_id, use_flagsDict, self._config_id) + # B = Build cpv use-flags config + # FIXME log_msg need a fix to log the use flags corect. + log_msg = "B %s:%s USE: %s Setup: %s" % (k, v['repo'], use_flagsDict, setup_id,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + i = i +1 + + def get_package_metadataDict(self, pkgdir, package_id): + # Make package_metadataDict + attDict = {} + package_metadataDict = {} + md_email_list = [] + # changelog_checksum_tree = portage.checksum.sha256hash(pkgdir + "/ChangeLog") + # changelog_text_tree = get_file_text(pkgdir + "/ChangeLog") + herd = None + pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd) + #metadata_xml_text_tree = get_file_text(pkgdir + "/metadata.xml") + # attDict['changelog_checksum'] = changelog_checksum_tree[0] + # attDict['changelog_text'] = changelog_text_tree + tmp_herds = pkg_md.herds() + if tmp_herds != (): + attDict['metadata_xml_herds'] = tmp_herds[0] + md_email_list.append(attDict['metadata_xml_herds'] + '@gentoo.org') + for maint in pkg_md.maintainers(): + md_email_list.append(maint.email) + if md_email_list != []: + attDict['metadata_xml_email'] = md_email_list + else: + log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml") + add_tbc_logs(self._session, log_msg, "qa", self._config_id) + attDict['metadata_xml_email'] = False + attDict['metadata_xml_checksum'] = portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0] + #attDict['metadata_xml_text'] = metadata_xml_text_tree + package_metadataDict[package_id] = attDict + return package_metadataDict + + def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree): + # Use packageDict to update the db + ebuild_id_list = add_new_ebuild_sql(self._session, packageDict) + + # Make old ebuilds unactive + for ebuild_id in ebuild_id_list: + new_ebuild_id_list.append(ebuild_id) + for ebuild_id in get_ebuild_id_list(self._session, package_id): + if not ebuild_id in new_ebuild_id_list: + if not ebuild_id in old_ebuild_id_list: + old_ebuild_id_list.append(ebuild_id) + if not old_ebuild_id_list == []: + add_old_ebuild(self._session, old_ebuild_id_list) + PackagesMetadataInfo = get_package_metadata_sql(self._session, package_id) + if PackagesMetadataInfo: + package_metadata_checksum_sql = PackagesMetadataInfo.Checksum + else: + package_metadata_checksum_sql = None + if package_metadata_checksum_sql is None or package_metadata_checksum_sql != package_metadataDict[package_id]['metadata_xml_checksum']: + update_package_metadata(self._session, package_metadataDict) + + # update the cp manifest checksum + update_manifest_sql(self._session, package_id, manifest_checksum_tree) + + # Get the best cpv for the configs and add it to config_cpv_listDict + PackageInfo, CategoryInfo, RepoInfo = get_package_info_from_package_id(self._session, package_id) + cp = CategoryInfo.Category + '/' + PackageInfo.Package + config_all_info = get_config_all_info(self._session) + config_list = [] + for config in get_config_all_info(self._session): + if config.Host is False: + config_list.append(config.ConfigId) + config_cpv_listDict = self.config_match_ebuild(cp, config_list) + + # Add the ebuild to the build jobs table if needed + self.add_new_build_job_db(ebuild_id_list, packageDict, config_cpv_listDict) + + def add_new_package_db(self, cp, repo): + # Add new categories package ebuild to tables package and ebuilds + # C = Checking + # N = New Package + log_msg = "C %s:%s" % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + log_msg = "N %s:%s" % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + repodir = self._myportdb.getRepositoryPath(repo) + pkgdir = repodir + "/" + cp # Get RepoDIR + cp + + # Get the cp manifest file checksum. + try: + manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0] + except: + manifest_checksum_tree = "0" + log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + log_msg = "C %s:%s ... Fail." % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + return None + package_id = add_new_package_sql(self._session, cp, repo) + + package_metadataDict = self.get_package_metadataDict(pkgdir, package_id) + # Get the ebuild list for cp + mytree = [] + mytree.append(repodir) + ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree) + if ebuild_list_tree == []: + log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + log_msg = "C %s:%s ... Fail." % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + return None + + # Make the needed packageDict with ebuild infos so we can add it later to the db. + packageDict ={} + new_ebuild_id_list = [] + old_ebuild_id_list = [] + for cpv in sorted(ebuild_list_tree): + packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo) + + self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree) + log_msg = "C %s:%s ... Done." % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + + def update_package_db(self, package_id): + # Update the categories and package with new info + # C = Checking + PackageInfo, CategoryInfo, RepoInfo = get_package_info_from_package_id(self._session, package_id) + cp = CategoryInfo.Category + '/' + PackageInfo.Package + repo = RepoInfo.Repo + log_msg = "C %s:%s" % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + repodir = self._myportdb.getRepositoryPath(repo) + pkgdir = repodir + "/" + cp # Get RepoDIR + cp + + # Get the cp mainfest file checksum + try: + manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0] + except: + manifest_checksum_tree = "0" + log_msg = "QA: Can't checksum the Manifest file. %s:%s" % (cp, repo,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + log_msg = "C %s:%s ... Fail." % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + return None + + # if we NOT have the same checksum in the db update the package + if manifest_checksum_tree != PackageInfo.Checksum: + + # U = Update + log_msg = "U %s:%s" % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + + # Get the ebuild list for cp + mytree = [] + mytree.append(repodir) + ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree) + if ebuild_list_tree == []: + log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + log_msg = "C %s:%s ... Fail." % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + return None + packageDict ={} + new_ebuild_id_list = [] + old_ebuild_id_list = [] + for cpv in sorted(ebuild_list_tree): + + # split out ebuild version + ebuild_version_tree = portage.versions.cpv_getversion(cpv) + + # Get packageDict for cpv + packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo) + + # Get the checksum of the ebuild in tree and db + ebuild_version_checksum_tree = packageDict[cpv]['checksum'] + checksums_db, fail= get_ebuild_checksums(self._session, package_id, ebuild_version_tree) + # check if we have dupes of the checksum from db + if checksums_db is None: + ebuild_version_manifest_checksum_db = None + elif fail: + dupe_ebuild_id_list = [] + for checksum in checksums_db: + ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id) + for ebuild_id in ebuilds_id: + log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,) + add_tbc_logs(self._session, log_msg, "error", self._config_id) + dupe_ebuild_id_list.append(ebuild_id) + add_old_ebuild(self._session, dupe_ebuild_id_list) + ebuild_version_manifest_checksum_db = None + else: + ebuild_version_manifest_checksum_db = checksums_db + + # Check if the checksum have change + if ebuild_version_manifest_checksum_db is None: + # N = New ebuild + log_msg = "N %s:%s" % (cpv, repo,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + elif ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db: + # U = Updated ebuild + log_msg = "U %s:%s" % (cpv, repo,) + add_tbc_logs(self._session, log_msg, "info", self._config_id) + else: + # Remove cpv from packageDict and add ebuild to new ebuils list + del packageDict[cpv] + ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id) + new_ebuild_id_list.append(ebuild_id) + package_metadataDict = self.get_package_metadataDict(pkgdir, package_id) + self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree) + + log_msg = "C %s:%s ... Done." % (cp, repo) + add_tbc_logs(self._session, log_msg, "info", self._config_id) diff --git a/tbc/pym/readconf.py b/tbc/pym/readconf.py new file mode 100644 index 0000000..db1056e --- /dev/null +++ b/tbc/pym/readconf.py @@ -0,0 +1,58 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import os +import sys +import re +from socket import getfqdn + +class get_conf_settings(object): +# open the /etc/tbc/tbc.conf file and get the needed +# settings for tbc + def __init__(self): + self.configfile = "/etc/tbc/tbc.conf" + + def read_tbc_settings_all(self): + # It will return a dict with options from the configfile + try: + open_conffile = open(self.configfile, 'r') + except: + sys.exit("Fail to open config file:" + self.configfile) + textlines = open_conffile.readlines() + for line in textlines: + element = line.split('=') + if element[0] == 'SQLBACKEND': # Databas backend + get_sql_backend = element[1] + if element[0] == 'SQLDB': # Database + get_sql_db = element[1] + if element[0] == 'SQLHOST': # Host + get_sql_host = element[1] + if element[0] == 'SQLUSER': # User + get_sql_user = element[1] + if element[0] == 'SQLPASSWD': # Password + get_sql_passwd = element[1] + # Buildhost root (dir for host/setup on host) + if element[0] == 'ZOBCSGITREPONAME': + get_tbc_gitreponame = element[1] + # Buildhost setup (host/setup on guest) + if element[0] == 'ZOBCSCONFIG': + get_tbc_config = element[1] + # if element[0] == 'LOGFILE': + # get_tbc_logfile = element[1] + open_conffile.close() + + tbc_settings_dict = {} + tbc_settings_dict['sql_backend'] = get_sql_backend.rstrip('\n') + tbc_settings_dict['sql_db'] = get_sql_db.rstrip('\n') + tbc_settings_dict['sql_host'] = get_sql_host.rstrip('\n') + tbc_settings_dict['sql_user'] = get_sql_user.rstrip('\n') + tbc_settings_dict['sql_passwd'] = get_sql_passwd.rstrip('\n') + tbc_settings_dict['tbc_gitreponame'] = get_tbc_gitreponame.rstrip('\n') + tbc_settings_dict['tbc_config'] = get_tbc_config.rstrip('\n') + tbc_settings_dict['hostname'] = getfqdn() + # tbc_settings_dict['tbc_logfile'] = get_tbc_logfile.rstrip('\n') + return tbc_settings_dict + +def read_config_settings(): + reader = get_conf_settings() + return reader.read_tbc_settings_all() diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py new file mode 100644 index 0000000..623f0b0 --- /dev/null +++ b/tbc/pym/sqlquerys.py @@ -0,0 +1,512 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +import datetime +from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Packages, Ebuilds, Repos, Categories, \ + Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs, BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \ + HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \ + BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \ + Keywords, PackagesMetadata, Emails, PackagesEmails, Setups +from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound +from sqlalchemy import and_, or_ + +# Guest Functions +def get_config_id(session, setup, host): + SetupInfo = session.query(Setups).filter_by(Setup = setup).one() + ConfigInfo = session.query(Configs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Hostname = host).one() + return ConfigInfo.ConfigId + +def add_tbc_logs(session, log_msg, log_type, config_id): + Add_Log = Logs(ConfigId = config_id, LogType = log_type, Msg = log_msg) + session.add(Add_Log) + session.commit() + +def update_deamon_status(session, status, config_id): + ConfigInfo = session.query(ConfigsMetaData).filter_by(ConfigId = config_id).one() + ConfigInfo.Status = status + session.commit() + +def get_jobs(session, config_id): + JobsInfo = session.query(Jobs).filter_by(Status = 'Waiting').filter_by(ConfigId = config_id).order_by(Jobs.JobId).all() + if JobsInfo == []: + return None + return JobsInfo + +def update_job_list(session, status, job_id): + JobInfo = session.query(Jobs).filter_by(JobId = job_id).one() + JobInfo.Status = status + session.commit() + +def get_config_all_info(session): + return session.query(Configs).all() + +def get_config_info(session, config_id): + ConfigInfo = session.query(Configs).filter_by(ConfigId = config_id).one() + return ConfigInfo + +def get_setup_info(session, config_id): + ConfigInfo = get_config_info(session, config_id) + SetupInfo = session.query(Setups).filter_by(SetupId = ConfigInfo.SetupId).one() + return SetupInfo + +def update_buildjobs_status(session, build_job_id, status, config_id): + BuildJobsInfo = session.query(BuildJobs).filter_by(BuildJobId = build_job_id).one() + BuildJobsInfo.Status = status + BuildJobsInfo.ConfigId = config_id + session.commit() + +def get_configmetadata_info(session, config_id): + return session.query(ConfigsMetaData).filter_by(ConfigId = config_id).one() + +def is_build_job_done(session, build_job_id): + try: + BuildJobsInfo = session.query(BuildJobs).filter_by(BuildJobId = build_job_id).one() + except NoResultFound as e: + return False + return True + +def get_packages_to_build(session, config_id): + SetupInfo = get_setup_info(session, config_id) + BuildJobsTmp = session.query(BuildJobs).filter(BuildJobs.SetupId==SetupInfo.SetupId). \ + order_by(BuildJobs.BuildJobId) + if session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(BuildNow = True).filter_by(Status = 'Waiting').all() == [] and session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Status = 'Waiting').all() == []: + return None + if not BuildJobsTmp.filter_by(BuildNow = True).first() is None: + BuildJobsInfo = session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(BuildNow = True). \ + filter_by(Status = 'Waiting').order_by(BuildJobs.BuildJobId).first() + else: + BuildJobsInfo = session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Status = 'Waiting').\ + order_by(BuildJobs.BuildJobId).first() + update_buildjobs_status(session, BuildJobsInfo.BuildJobId, 'Looked', config_id) + EbuildsInfo = session.query(Ebuilds).filter_by(EbuildId = BuildJobsInfo.EbuildId).one() + PackagesInfo, CategoriesInfo = session.query(Packages, Categories).filter(Packages.PackageId==EbuildsInfo.PackageId).filter(Packages.CategoryId==Categories.CategoryId).one() + ReposInfo = session.query(Repos).filter_by(RepoId = PackagesInfo.RepoId).one() + uses={} + for BuildJobsUseInfo, UsesInfo in session.query(BuildJobsUse, Uses).filter(BuildJobsUse.BuildJobId==BuildJobsInfo.BuildJobId).filter(BuildJobsUse.UseId==Uses.UseId).all(): + uses[UsesInfo.Flag] = BuildJobsUseInfo.Status + if uses == {}: + uses = None + emerge_options_list = [] + for ConfigsEmergeOptionsInfo, EmergeOptionsInfo in session.query(ConfigsEmergeOptions, EmergeOptions). \ + filter(ConfigsEmergeOptions.ConfigId==config_id). \ + filter(ConfigsEmergeOptions.EOptionId==EmergeOptions.EmergeOptionId).all(): + emerge_options_list.append(EmergeOptionsInfo.EOption) + build_dict={} + build_dict['config_id'] = config_id + build_dict['setup_id'] = BuildJobsInfo.SetupId + build_dict['build_job_id'] = BuildJobsInfo.BuildJobId + build_dict['ebuild_id']= EbuildsInfo.EbuildId + build_dict['package_id'] = EbuildsInfo.PackageId + build_dict['package'] = PackagesInfo.Package + build_dict['category'] = CategoriesInfo.Category + build_dict['repo'] = ReposInfo.Repo + build_dict['removebin'] = BuildJobsInfo.RemoveBin + build_dict['ebuild_version'] = EbuildsInfo.Version + build_dict['checksum'] = EbuildsInfo.Checksum + build_dict['cp'] = CategoriesInfo.Category + '/' + PackagesInfo.Package + build_dict['cpv'] = build_dict['cp'] + '-' + EbuildsInfo.Version + build_dict['build_useflags'] = uses + build_dict['emerge_options'] = emerge_options_list + return build_dict + +def get_category_info(session, category): + try: + CategoryInfo = session.query(Categories).filter_by(Category = category).filter_by(Active = True).one() + except NoResultFound as e: + return False + return CategoryInfo + +def get_repo_info(session, repo): + try: + RepoInfo = session.query(Repos).filter_by(Repo = repo).one() + except NoResultFound as e: + return False + return RepoInfo + +def get_package_info(session, category, package, repo): + CategoryInfo = get_category_info(session, category) + RepoInfo = get_repo_info(session, repo) + try: + PackageInfo = session.query(Packages).filter_by(CategoryId = CategoryInfo.CategoryId). \ + filter_by(Package = package).filter_by(RepoId = RepoInfo.RepoId).filter_by(Active = True).one() + except NoResultFound as e: + return False + return PackageInfo + +def get_ebuild_info(session, build_dict): + EbuildInfo = session.query(Ebuilds).filter_by(Version = build_dict['ebuild_version']).filter_by(Checksum = build_dict['checksum']).\ + filter_by(PackageId = build_dict['package_id']).filter_by(Active = True) + if EbuildInfo.all() == []: + return None, True + try: + EbuildInfo2 = EbuildInfo.one() + except (MultipleResultsFound) as e: + return EbuildInfo.all(), True + return EbuildInfo2, False + +def get_build_job_id(session, build_dict): + BuildJobsIdInfo = session.query(BuildJobs.BuildJobId).filter_by(EbuildId = build_dict['ebuild_id']).filter_by(ConfigId = build_dict['config_id']).all() + if BuildJobsIdInfo == []: + return None + for build_job_id in BuildJobsIdInfo: + BuildJobsUseInfo = session.query(BuildJobsUse).filter_by(BuildJobId = build_job_id.BuildJobId).all() + useflagsdict = {} + if BuildJobsUseInfo == []: + useflagsdict = None + else: + for x in BuildJobsUseInfo: + useflagsdict[x.UseId] = x.Status + if useflagsdict == build_dict['build_useflags']: + return build_job_id.BuildJobId + return None + +def get_use_id(session, use_flag): + try: + UseIdInfo = session.query(Uses).filter_by(Flag = use_flag).one() + except NoResultFound as e: + return None + return UseIdInfo.UseId + +def get_hilight_info(session): + return session.query(HiLight).all() + +def get_error_info_list(session): + return session.query(ErrorsInfo).all() + +def add_e_info(session, emerge_info): + AddEmergeInfo = EmergeInfo(EmergeInfoText = emerge_info) + session.add(AddEmergeInfo) + session.flush() + EmergeInfoId = AddEmergeInfo.EInfoId + session.commit() + return EmergeInfoId + +def del_old_build_jobs(session, build_job_id): + session.query(BuildJobsUse).filter(BuildJobsUse.BuildJobId == build_job_id).delete() + session.query(BuildJobsRedo).filter(BuildJobsRedo.BuildJobId == build_job_id).delete() + session.query(BuildJobsEmergeOptions).filter(BuildJobsEmergeOptions.BuildJobId == build_job_id).delete() + session.query(BuildJobs).filter(BuildJobs.BuildJobId == build_job_id).delete() + session.commit() + +def add_new_buildlog(session, build_dict, build_log_dict): + build_log_id_list = session.query(BuildLogs.BuildLogId).filter_by(EbuildId = build_dict['ebuild_id']).all() + + def add_new_hilight(log_id, build_log_dict): + for k, hilight_tmp in sorted(build_log_dict['hilight_dict'].items()): + NewHiLight = BuildLogsHiLight(LogId = log_id, StartLine = hilight_tmp['startline'], EndLine = hilight_tmp['endline'], HiLightCssId = hilight_tmp['hilight_css_id']) + session.add(NewHiLight) + session.commit() + + def build_log_id_match(build_log_id_list, build_dict, build_log_dict): + for build_log_id in build_log_id_list: + log_hash = session.query(BuildLogs.LogHash).filter_by(BuildLogId = build_log_id[0]).one() + use_list = session.query(BuildLogsUse).filter_by(BuildLogId = build_log_id[0]).all() + useflagsdict = {} + if use_list == []: + useflagsdict = None + else: + for use in use_list: + useflagsdict[use.UseId] = use.Status + if log_hash[0] == build_log_dict['log_hash'] and build_dict['build_useflags'] == useflagsdict: + if session.query(BuildLogsConfig).filter(BuildLogsConfig.ConfigId.in_([build_dict['config_id']])).filter_by(BuildLogId = build_log_id[0]): + return None, True + e_info_id = add_e_info(session, build_log_dict['emerge_info']) + NewBuildLogConfig = BuildLogsConfig(BuildLogId = build_log_id[0], ConfigId = build_dict['config_id'], LogName = build_log_dict['logfilename'], EInfoId = e_info_id) + session.add(NewBuildLogConfig) + session.commit() + return build_log_id[0], True + return None, False + + def build_log_id_no_match(build_dict, build_log_dict): + if build_log_dict['summary_error_list'] == []: + NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = False, SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash']) + else: + NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = True, SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash']) + session.add(NewBuildLog) + session.flush() + build_log_id = NewBuildLog.BuildLogId + session.commit() + if build_log_dict['summary_error_list'] != []: + for error in build_log_dict['summary_error_list']: + NewError = BuildLogsErrors(BuildLogId = build_log_id, ErrorId = error) + session.add(NewError) + session.commit() + e_info_id = add_e_info(session, build_log_dict['emerge_info']) + NewBuildLogConfig = BuildLogsConfig(BuildLogId = build_log_id, ConfigId = build_dict['config_id'], LogName = build_log_dict['logfilename'], EInfoId = e_info_id) + session.add(NewBuildLogConfig) + session.flush() + log_id = NewBuildLogConfig.LogId + session.commit() + add_new_hilight(log_id, build_log_dict) + if not build_dict['build_useflags'] is None: + for use_id, status in build_dict['build_useflags'].items(): + NewBuildLogUse = BuildLogsUse(BuildLogId = build_log_id, UseId = use_id, Status = status) + session.add(NewBuildLogUse) + session.flush() + session.commit() + return build_log_id + + if build_dict['build_job_id'] is None and build_log_id_list == []: + build_log_id = build_log_id_no_match(build_dict, build_log_dict) + return build_log_id + elif build_dict['build_job_id'] is None and build_log_id_list != []: + build_log_id, match = build_log_id_match(build_log_id_list, build_dict, build_log_dict) + if not match: + build_log_id = build_log_id_no_match(build_dict, build_log_dict) + return build_log_id + elif not build_dict['build_job_id'] is None and build_log_id_list != []: + build_log_id, match = build_log_id_match(build_log_id_list, build_dict, build_log_dict) + if not match: + build_log_id = build_log_id_no_match(build_dict, build_log_dict) + del_old_build_jobs(session, build_dict['build_job_id']) + return build_log_id + elif not build_dict['build_job_id'] is None and build_log_id_list == []: + build_log_id = build_log_id_no_match(build_dict, build_log_dict) + del_old_build_jobs(session, build_dict['build_job_id']) + return build_log_id + +def update_fail_times(session, FailInfo): + NewBuildJobs = session.query(BuildJobs).filter_by(BuildJobId = FailInfo.BuildJobId).one() + NewBuildJobs.TimeStamp = datetime.datetime.utcnow() + session.commit() + +def get_fail_times(session, build_dict): + try: + FailInfo = session.query(BuildJobsRedo).filter_by(BuildJobId = build_dict['build_job_id']).filter_by(FailType = build_dict['type_fail']).one() + except NoResultFound as e: + return False + return True + +def add_fail_times(session, fail_querue_dict): + print(fail_querue_dict) + NewBuildJobsRedo = BuildJobsRedo(BuildJobId = fail_querue_dict['build_job_id'], FailType = fail_querue_dict['fail_type'], FailTimes = fail_querue_dict['fail_times']) + session.add(NewBuildJobsRedo) + session.commit() + +# Host Functions +def update_repo_db(session, repo_list): + for repo in repo_list: + if not get_repo_info(session, repo): + session.add(Repos(Repo = repo)) + session.commit() + +def update_categories_db(session, category): + if not get_category_info(session, category): + session.add(Categories(Category = category)) + session.commit() + +def get_keyword_id(session, keyword): + try: + KeywordsInfo = session.query(Keywords).filter_by(Keyword = keyword).one() + except NoResultFound as e: + return None + return KeywordsInfo.KeywordId + +def add_new_ebuild_metadata_sql(session, ebuild_id, keywords, restrictions, iuse_list): + for restriction in restrictions: + if restriction in ["!"]: + restriction = restriction[1:] + if restriction in ["?"]: + restriction = restriction[:1] + if restriction != '(' or restriction != ')': + try: + RestrictionInfo = session.query(Restrictions).filter_by(Restriction = restriction).one() + except NoResultFound as e: + session.add(Restrictions(Restriction = restriction)) + session.commit() + RestrictionInfo = session.query(Restrictions).filter_by(Restriction = restriction).one() + session.add(EbuildsRestrictions(EbuildId = ebuild_id, RestrictionId = RestrictionInfo.RestrictionId)) + session.commit() + for iuse in iuse_list: + status = False + if iuse[0] in ["+"]: + iuse = iuse[1:] + status = True + elif iuse[0] in ["-"]: + iuse = iuse[1:] + use_id = get_use_id(session, iuse) + if use_id is None: + session.add(Uses(Flag = iuse)) + session.commit() + use_id = get_use_id(session, iuse) + session.add(EbuildsIUse(EbuildId = ebuild_id, UseId = use_id, Status = status)) + session.commit() + for keyword in keywords: + status = 'Stable' + if keyword[0] in ["~"]: + keyword = keyword[1:] + status = 'Unstable' + elif keyword[0] in ["-"]: + keyword = keyword[1:] + status = 'Negative' + keyword_id = get_keyword_id(session, keyword) + if keyword_id is None: + session.add(Keywords(Keyword = keyword)) + session.commit() + keyword_id = get_keyword_id(session, keyword) + session.add(EbuildsKeywords(EbuildId = ebuild_id, KeywordId = keyword_id, Status = status)) + session.commit() + +def add_new_ebuild_sql(session, packageDict): + ebuild_id_list = [] + for k, v in packageDict.items(): + session.add(Ebuilds(PackageId = v['package_id'], Version = v['ebuild_version'], Checksum = v['checksum'], Active = True)) + session.flush() + try: + EbuildInfo = session.query(Ebuilds).filter_by(Version = v['ebuild_version']).filter_by(Checksum = v['checksum']).\ + filter_by(PackageId = v['package_id']).filter_by(Active = True).one() + except (MultipleResultsFound) as e: + for x in session.query(Ebuilds).filter_by(Version = v['ebuild_version']).filter_by(Checksum = v['checksum']).\ + filter_by(PackageId = v['package_id']).filter_by(Active = True).all(): + print(x.EbuildId) + # FIXME + #x.Checksum = 0 + #x.Active = False + #session.commit() + sys.exit() + session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, Revision = v['ebuild_version_revision_tree'])) + session.commit() + ebuild_id_list.append(EbuildInfo.EbuildId) + restrictions = [] + keywords = [] + iuse = [] + for i in v['ebuild_version_metadata_tree'][4].split(): + restrictions.append(i) + for i in v['ebuild_version_metadata_tree'][8].split(): + keywords.append(i) + for i in v['ebuild_version_metadata_tree'][10].split(): + iuse.append(i) + add_new_ebuild_metadata_sql(session, EbuildInfo.EbuildId, keywords, restrictions, iuse) + return ebuild_id_list + +def get_ebuild_id_list(session, package_id): + ebuild_id_list = [] + for EbuildInfo in session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Active = True).all(): + ebuild_id_list.append(EbuildInfo.EbuildId) + return ebuild_id_list + +def get_build_job_all(session, ebuild_id): + return session.query(BuildJobs).filter_by(EbuildId = ebuild_id).all() + +def add_old_ebuild(session, old_ebuild_list): + for ebuild_id in old_ebuild_list: + EbuildInfo = session.query(Ebuilds).filter_by(EbuildId = ebuild_id).one() + EbuildInfo.Active = False + session.commit() + build_job_id_list = get_build_job_all(session, ebuild_id) + if build_job_id_list != []: + for build_job in build_job_id_list: + del_old_build_jobs(session, build_job.BuildJobId) + +def add_new_package_sql(session, cp, repo): + element = cp.split('/') + categories = element[0] + package = element[1] + RepoInfo =get_repo_info(session, repo) + repo_id = RepoInfo.RepoId + CategoriesInfo = get_category_info(session, categories) + category_id = CategoriesInfo.CategoryId + session.add(Packages(Package = package, CategoryId = category_id, RepoId = repo_id, Checksum = '0', Active = True)) + session.commit() + PackageInfo = get_package_info(session, categories, package, repo) + return PackageInfo.PackageId + +def get_package_metadata_sql(session, package_id): + try: + PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = package_id).one() + except NoResultFound as e: + return False + return PackagesMetadataInfo + +def update_email_info(session, email): + try: + EmailInfo = session.query(Emails).filter_by(Email = email).one() + except NoResultFound as e: + session.add(Emails(Email = email)) + session.commit() + EmailInfo = session.query(Emails).filter_by(Email = email).one() + return EmailInfo + +def update_package_email_info(session, email_id, package_id): + try: + PackagesEmailInfo = session.query(PackagesEmails).filter_by(EmailId = email_id).filter_by(PackageId = package_id).one() + except NoResultFound as e: + session.add(PackagesEmails(EmailId = email_id, PackageId = package_id)) + session.commit() + PackagesEmailInfo = session.query(PackagesEmails).filter_by(EmailId = email_id).filter_by(PackageId = package_id).one() + return PackagesEmailInfo + +def update_package_metadata(session, package_metadataDict): + for k, v in package_metadataDict.items(): + try: + PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = k).one() + except NoResultFound as e: + session.add(PackagesMetadata(PackageId = k, Checksum = v['metadata_xml_checksum'])) + session.commit() + else: + PackagesMetadataInfo.Checksum = v['metadata_xml_checksum'] + session.commit() + if v['metadata_xml_email']: + for email in v['metadata_xml_email']: + EmailInfo = update_email_info(session, email) + PackagesEmailInfo = update_package_email_info(session, EmailInfo.EmailId, k) + +def update_manifest_sql(session, package_id, manifest_checksum_tree): + PackagesInfo = session.query(Packages).filter_by(PackageId = package_id).one() + PackagesInfo.Checksum = manifest_checksum_tree + session.commit() + +def get_package_info_from_package_id(session, package_id): + PackageInfo = session.query(Packages).filter_by(PackageId = package_id).one() + CategoryInfo = session.query(Categories).filter_by(CategoryId = PackageInfo.CategoryId).one() + RepoInfo = session.query(Repos).filter_by(RepoId = PackageInfo.RepoId).one() + return PackageInfo, CategoryInfo, RepoInfo + +def add_new_build_job(session, ebuild_id, setup_id, use_flagsDict, config_id): + NewBuildJobs =BuildJobs(EbuildId = ebuild_id, SetupId = setup_id, ConfigId = config_id, Status = 'Waiting', BuildNow = False, RemoveBin = True) + session.add(NewBuildJobs) + session.flush() + build_job_id = NewBuildJobs.BuildJobId + session.commit() + for k, v in use_flagsDict.items(): + use_id = get_use_id(session, k) + session.add(BuildJobsUse(BuildJobId = build_job_id, UseId = use_id, Status = v)) + session.commit() + +def get_ebuild_checksums(session, package_id, ebuild_version): + ebuild_checksum_list = [] + try: + EbuildInfo = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Version = ebuild_version).filter_by(Active = True).one() + except NoResultFound as e: + return None, False + except MultipleResultsFound as e: + EbuildInfo2 = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Version = ebuild_version).filter_by(Active = True).all() + for Ebuild in EbuildInfo2: + print("ebuild version checksum") + print(ebuild_version) + print(Ebuild.Version) + print(Ebuild.Checksum) + ebuild_checksum_list.append(Ebuild.Checksum) + return ebuild_checksum_list, True + return EbuildInfo.Checksum, False + +def get_ebuild_id_db(session, checksum, package_id): + try: + EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).one() + except NoResultFound as e: + return None, True + except MultipleResultsFound as e: + EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).all() + ebuilds_id = [] + for EbuildInfo in EbuildInfos: + ebuilds_id.append(EbuildInfo.EbuildId) + return ebuilds_id, True + return EbuildInfos.EbuildId, False + +def check_host_updatedb(session): + try: + JobsInfo = session.query(Jobs).filter_by(Status = 'Done').filter_by(JobType = 'esync').one() + except NoResultFound as e: + return True + return False diff --git a/tbc/pym/sync.py b/tbc/pym/sync.py new file mode 100644 index 0000000..a75562f --- /dev/null +++ b/tbc/pym/sync.py @@ -0,0 +1,73 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +import portage +import os +import errno +import sys +import time + +from _emerge.main import emerge_main +from tbc.readconf import get_conf_settings +from tbc.sqlquerys import get_config_id, add_tbc_logs, get_config_all_info, get_configmetadata_info +from tbc.updatedb import update_db_main +from tbc.readconf import read_config_settings + +def sync_tree(session): + tbc_settings_dict = read_config_settings() + _hostname = tbc_settings_dict['hostname'] + _config = tbc_settings_dict['tbc_config'] + config_id = get_config_id(session, _config, _hostname) + host_config = _hostname +"/" + _config + default_config_root = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/" + mysettings = portage.config(config_root = default_config_root) + GuestBusy = True + log_msg = "Waiting for Guest to be idel" + add_tbc_logs(session, log_msg, "info", config_id) + guestid_list = [] + for config in get_config_all_info(session): + if not config.Host: + guestid_list.append(config.ConfigId) + while GuestBusy: + Status_list = [] + for guest_id in guestid_list: + ConfigMetadata = get_configmetadata_info(session, guest_id) + Status_list.append(ConfigMetadata.Status) + if not 'Runing' in Status_list: + GuestBusy = False + time.sleep(30) + try: + os.remove(mysettings['PORTDIR'] + "/profiles/config/parent") + os.rmdir(mysettings['PORTDIR'] + "/profiles/config") + except: + pass + tmpcmdline = [] + tmpcmdline.append("--sync") + tmpcmdline.append("--quiet") + tmpcmdline.append("--config-root=" + default_config_root) + log_msg = "Emerge --sync" + add_tbc_logs(session, log_msg, "info", config_id) + fail_sync = emerge_main(args=tmpcmdline) + if fail_sync: + log_msg = "Emerge --sync fail!" + add_tbc_logs(session, log_msg, "error", config_id) + return False + else: + # Need to add a config dir so we can use profiles/base for reading the tree. + # We may allready have the dir on local repo when we sync. + try: + os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777) + with open(mysettings['PORTDIR'] + "/profiles/config/parent", "w") as f: + f.write("../base\n") + f.close() + except: + pass + log_msg = "Emerge --sync ... Done." + add_tbc_logs(session, log_msg, "info", config_id) + result = update_db_main(session, config_id) + if result: + return True + else: + log_msg = "Updatedb fail" + add_tbc_logs(session, log_msg, "info", config_id) diff --git a/tbc/pym/text.py b/tbc/pym/text.py new file mode 100644 index 0000000..c78c432 --- /dev/null +++ b/tbc/pym/text.py @@ -0,0 +1,49 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +import sys +import re +import os +import errno +from portage.util import grablines + +def get_file_text(filename): + # Return the filename contents + try: + textfile = open(filename, encoding='utf-8') + except: + return "No file", filename + text = "" + for line in textfile: + text += line + textfile.close() + return text + +def get_ebuild_cvs_revision(filename): + """Return the ebuild contents""" + try: + ebuildfile = open(filename, encoding='utf-8') + except: + return "No Ebuild file there" + text = "" + dataLines = ebuildfile.readlines() + for i in dataLines: + text = text + i + " " + line2 = dataLines[2] + field = line2.split(" ") + ebuildfile.close() + try: + cvs_revision = field[3] + except: + cvs_revision = '' + return cvs_revision + +def get_log_text_dict(filename): + """Return the log contents as a dict""" + logfile_dict = {} + index = 1 + for text_line in grablines(filename): + logfile_dict[index] = text_line + index = index + 1 + return logfile_dict, index - 1 diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py new file mode 100644 index 0000000..12b94c5 --- /dev/null +++ b/tbc/pym/updatedb.py @@ -0,0 +1,135 @@ +# Copyright 1998-2015 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function +import sys +import os +import multiprocessing +import time +import portage +from sqlalchemy.orm import scoped_session, sessionmaker +from tbc.ConnectionManager import NewConnection +from tbc.sqlquerys import add_tbc_logs, get_package_info, update_repo_db, \ + update_categories_db, get_configmetadata_info, get_config_all_info, add_new_build_job, \ + get_config_info +from tbc.check_setup import check_make_conf +from tbc.package import tbc_package +# Get the options from the config file set in tbc.readconf +from tbc.readconf import get_conf_settings + +def init_portage_settings(session, config_id, tbc_settings_dict): + # check config setup + check_make_conf(session, config_id, tbc_settings_dict) + log_msg = "Check configs done" + add_tbc_logs(session, log_msg, "info", config_id) + + # Get default config from the configs table and default_config=1 + host_config = tbc_settings_dict['hostname'] +"/" + tbc_settings_dict['tbc_config'] + default_config_root = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/" + + # Set config_root (PORTAGE_CONFIGROOT) to default_config_root + mysettings = portage.config(config_root = default_config_root) + log_msg = "Setting default config to: %s" % (host_config,) + add_tbc_logs(session, log_msg, "info", config_id) + return mysettings + +def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings_dict, config_id): + session_factory = sessionmaker(bind=NewConnection(tbc_settings_dict)) + Session = scoped_session(session_factory) + session2 = Session() + init_package = tbc_package(session2, mysettings, myportdb, config_id, tbc_settings_dict) + + # split the cp to categories and package + element = cp.split('/') + categories = element[0] + package = element[1] + + # update the categories table + update_categories_db(session2, categories) + + # Check if we have the cp in the package table + PackagesInfo = get_package_info(session2, categories, package, repo) + if PackagesInfo: + # Update the packages with ebuilds + init_package.update_package_db(PackagesInfo.PackageId) + else: + # Add new package with ebuilds + init_package.add_new_package_db(cp, repo) + Session.remove() + +def update_cpv_db(session, config_id, tbc_settings_dict): + GuestBusy = True + log_msg = "Waiting for Guest to be idel" + add_tbc_logs(session, log_msg, "info", config_id) + guestid_list = [] + for config in get_config_all_info(session): + if not config.Host: + guestid_list.append(config.ConfigId) + while GuestBusy: + Status_list = [] + for guest_id in guestid_list: + ConfigMetadata = get_configmetadata_info(session, guest_id) + Status_list.append(ConfigMetadata.Status) + if not 'Runing' in Status_list: + break + time.sleep(30) + + mysettings = init_portage_settings(session, config_id, tbc_settings_dict) + log_msg = "Checking categories, package, ebuilds" + add_tbc_logs(session, log_msg, "info", config_id) + new_build_jobs_list = [] + + # Setup portdb, package + myportdb = portage.portdbapi(mysettings=mysettings) + repo_list = () + repos_trees_list = [] + + # Use all cores when multiprocessing + pool_cores= multiprocessing.cpu_count() + pool = multiprocessing.Pool(processes=pool_cores) + + # Will run some update checks and update package if needed + + # Get the repos and update the repos db + repo_list = myportdb.getRepositories() + update_repo_db(session, repo_list) + + # Close the db for the multiprocessing pool will make new ones + # and we don't need this one for some time. + + # Get the rootdirs for the repos + repo_trees_list = myportdb.porttrees + for repo_dir in repo_trees_list: + repo = myportdb.getRepositoryName(repo_dir) + repo_dir_list = [] + repo_dir_list.append(repo_dir) + + # Get the package list from the repo + package_list_tree = myportdb.cp_all(trees=repo_dir_list) + + # Run the update package for all package in the list and in a multiprocessing pool + for cp in sorted(package_list_tree): + pool.apply_async(update_cpv_db_pool, (mysettings, myportdb, cp, repo, tbc_settings_dict, config_id,)) + # use this when debuging + #update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings_dict, config_id) + + #close and join the multiprocessing pools + pool.close() + pool.join() + log_msg = "Checking categories, package and ebuilds ... done" + add_tbc_logs(session, log_msg, "info", config_id) + +def update_db_main(session, config_id): + # Main + + # Logging + reader = get_conf_settings() + tbc_settings_dict=reader.read_tbc_settings_all() + log_msg = "Update db started." + add_tbc_logs(session, log_msg, "info", config_id) + + # Update the cpv db + update_cpv_db(session, config_id, tbc_settings_dict) + log_msg = "Update db ... Done." + add_tbc_logs(session, log_msg, "info", config_id) + return True