commit: 3c864667205655ada153bd6828c67253c0d30da1 Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> AuthorDate: Fri Jul 17 21:31:30 2015 +0000 Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> CommitDate: Fri Jul 17 21:31:30 2015 +0000 URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3c864667
move git repopath to db conf/tbc.conf | 6 +--- sql/structure_dump.sql | 47 ++++++++--------------------- tbc/bin/tbc_guest_jobs | 21 ++++++------- tbc/bin/tbc_host_jobs | 13 +++----- tbc/pym/build_depgraph.py | 14 ++++----- tbc/pym/build_job.py | 4 +-- tbc/pym/build_log.py | 24 ++++++++++----- tbc/pym/check_setup.py | 22 +++++--------- tbc/pym/db_mapping.py | 12 +++----- tbc/pym/package.py | 45 +++++++++++++-------------- tbc/pym/qachecks.py | 18 +++++------ tbc/pym/readconf.py | 77 ++++++++++++++++++----------------------------- tbc/pym/sqlquerys.py | 35 +++++++++++---------- tbc/pym/sync.py | 23 ++++++++------ tbc/pym/updatedb.py | 44 +++++++++++++++------------ 15 files changed, 183 insertions(+), 222 deletions(-) diff --git a/conf/tbc.conf b/conf/tbc.conf index 339a22a..88fb9ec 100644 --- a/conf/tbc.conf +++ b/conf/tbc.conf @@ -8,8 +8,4 @@ SQLHOST=localhost SQLUSER=buildhost # Sql Password SQLPASSWD=buildhost -# TBC git repo path for the setups/configs /etc/portage -TBCGITREPOPATH=/var/cache/tbc/tinderboxs_configs -TBCSONFIG=base -# Main logfile -LOGFILE=/var/log/tbc + diff --git a/sql/structure_dump.sql b/sql/structure_dump.sql index e418964..459f6bf 100644 --- a/sql/structure_dump.sql +++ b/sql/structure_dump.sql @@ -3,9 +3,9 @@ -- http://www.phpmyadmin.net -- -- Host: localhost --- Generation Time: May 09, 2015 at 03:13 PM +-- Generation Time: Jul 17, 2015 at 08:35 PM -- Server version: 10.0.15-MariaDB-log --- PHP Version: 5.6.5-pl0-gentoo +-- PHP Version: 5.6.10-pl0-gentoo SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; SET time_zone = "+00:00"; @@ -111,6 +111,8 @@ CREATE TABLE IF NOT EXISTS `build_logs` ( `build_log_id` int(11) NOT NULL, `ebuild_id` int(11) NOT NULL, `fail` tinyint(1) NOT NULL DEFAULT '0', + `rmqa` tinyint(1) NOT NULL, + `others` tinyint(1) NOT NULL, `summery_text` longtext NOT NULL, `log_hash` varchar(100) NOT NULL, `bug_id` int(10) NOT NULL DEFAULT '0', @@ -173,22 +175,10 @@ CREATE TABLE IF NOT EXISTS `build_logs_hilight` ( -- -------------------------------------------------------- -- --- Table structure for table `build_logs_qa` +-- Table structure for table `build_logs_repoman_qa` -- -CREATE TABLE IF NOT EXISTS `build_logs_qa` ( -`id` int(11) NOT NULL, - `build_log_id` int(11) NOT NULL, - `summery_text` text NOT NULL -) ENGINE=MyISAM DEFAULT CHARSET=utf8; - --- -------------------------------------------------------- - --- --- Table structure for table `build_logs_repoman` --- - -CREATE TABLE IF NOT EXISTS `build_logs_repoman` ( +CREATE TABLE IF NOT EXISTS `build_logs_repoman_qa` ( `id` int(11) NOT NULL, `build_log_id` int(11) NOT NULL, `summery_text` text NOT NULL @@ -254,7 +244,6 @@ CREATE TABLE IF NOT EXISTS `configs_emerge_options` ( CREATE TABLE IF NOT EXISTS `configs_metadata` ( `id` int(11) NOT NULL, `config_id` int(11) NOT NULL, - `profile` varchar(50) NOT NULL, `keyword_id` int(11) NOT NULL, `make_conf_text` text NOT NULL, `checksum` varchar(100) NOT NULL, @@ -264,7 +253,7 @@ CREATE TABLE IF NOT EXISTS `configs_metadata` ( `updateing` tinyint(1) NOT NULL, `status` enum('Waiting','Runing','Stoped') NOT NULL, `auto` tinyint(1) NOT NULL, - `git_www` varchar(100) NOT NULL COMMENT 'git repo www wiev address', + `repo_path` varchar(100) NOT NULL COMMENT 'git repo path for etc/portage', `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='Config Status'; @@ -318,7 +307,8 @@ CREATE TABLE IF NOT EXISTS `ebuilds_keywords` ( CREATE TABLE IF NOT EXISTS `ebuilds_metadata` ( `id` int(11) NOT NULL, `ebuild_id` int(11) NOT NULL, - `revision` varchar(10) NOT NULL COMMENT 'CVS revision' + `revision` varchar(10) NOT NULL COMMENT 'CVS revision', + `descriptions` varchar(200) NOT NULL ) ENGINE=MyISAM DEFAULT CHARSET=utf8; -- -------------------------------------------------------- @@ -591,15 +581,9 @@ ALTER TABLE `build_logs_hilight` ADD PRIMARY KEY (`id`), ADD KEY `log_id` (`log_id`), ADD KEY `hilight_id` (`hilight_css_id`), ADD KEY `hilight_css_id` (`hilight_css_id`); -- --- Indexes for table `build_logs_qa` --- -ALTER TABLE `build_logs_qa` - ADD PRIMARY KEY (`id`), ADD KEY `build_log_id` (`build_log_id`); - --- --- Indexes for table `build_logs_repoman` +-- Indexes for table `build_logs_repoman_qa` -- -ALTER TABLE `build_logs_repoman` +ALTER TABLE `build_logs_repoman_qa` ADD PRIMARY KEY (`id`), ADD KEY `build_logs_id` (`build_log_id`); -- @@ -808,14 +792,9 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; ALTER TABLE `build_logs_hilight` MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; -- --- AUTO_INCREMENT for table `build_logs_qa` --- -ALTER TABLE `build_logs_qa` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; --- --- AUTO_INCREMENT for table `build_logs_repoman` +-- AUTO_INCREMENT for table `build_logs_repoman_qa` -- -ALTER TABLE `build_logs_repoman` +ALTER TABLE `build_logs_repoman_qa` MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; -- -- AUTO_INCREMENT for table `build_logs_use` diff --git a/tbc/bin/tbc_guest_jobs b/tbc/bin/tbc_guest_jobs index 4f5716b..a4e1382 100755 --- a/tbc/bin/tbc_guest_jobs +++ b/tbc/bin/tbc_guest_jobs @@ -6,7 +6,7 @@ from __future__ import print_function from tbc.readconf import read_config_settings from tbc.ConnectionManager import NewConnection -from tbc.sqlquerys import add_logs, get_config_id, check_host_updatedb, update_deamon_status +from tbc.sqlquerys import add_logs, get_config_id_fqdn, check_host_updatedb, update_deamon_status from tbc.check_setup import check_configure_guest from tbc.build_job import build_job_action from tbc.jobs import jobs_main @@ -18,26 +18,23 @@ import time def main(): repeat = True - tbc_settings_dict = read_config_settings() - config = tbc_settings_dict['tbc_config'] - hostname = tbc_settings_dict['hostname'] - Session = sessionmaker(bind=NewConnection(tbc_settings_dict)) + tbc_settings = read_config_settings() + Session = sessionmaker(bind=NewConnection(tbc_settings)) session = Session() - config_id = get_config_id(session, config, hostname) + config_id = get_config_id_fqdn(session, tbc_settings['hostname']) add_logs(session, "Job and build deamon started.", "info", config_id) + update_deamon_status(session, 'Runing', config_id) init_build_job = build_job_action(config_id, session) while repeat: jobs_main(session, config) - if not check_configure_guest(session, tbc_settings_dict, config_id) or check_host_updatedb(session): - time.sleep(30) + if not check_configure_guest(session, config_id) or check_host_updatedb(session): + time.sleep(60) continue else: - status = 'Runing' - update_deamon_status(session, status, config_id) + update_deamon_status(session, 'Runing', config_id) init_build_job.procces_build_jobs() time.sleep(5) - status = 'Waiting' - update_deamon_status(session, status, config_id) + update_deamon_status(session, 'Waiting', config_id) conn.close if __name__ == "__main__": diff --git a/tbc/bin/tbc_host_jobs b/tbc/bin/tbc_host_jobs index 21f9d31..c9bbf8c 100755 --- a/tbc/bin/tbc_host_jobs +++ b/tbc/bin/tbc_host_jobs @@ -5,22 +5,19 @@ from __future__ import print_function -from tbc.readconf import get_conf_settings +from tbc.readconf import read_config_settings from tbc.jobs import jobs_main from tbc.ConnectionManager import NewConnection -from tbc.sqlquerys import add_logs, get_config_id +from tbc.sqlquerys import add_logs, get_config_id_fqdn from sqlalchemy.orm import sessionmaker import time def main(): # Main - reader = get_conf_settings() - tbc_settings_dict=reader.read_tbc_settings_all() - config = tbc_settings_dict['tbc_config'] - hostname = tbc_settings_dict['hostname'] - Session = sessionmaker(bind=NewConnection(tbc_settings_dict)) + tbc_settings = read_config_settings() + Session = sessionmaker(bind=NewConnection(tbc_settings)) session = Session() - config_id = get_config_id(session, config, hostname) + config_id = get_config_id_fqdn(session, tbc_settings['hostname']) add_logs(session, "Job deamon started", "info", config_id) repeat = True while repeat: diff --git a/tbc/pym/build_depgraph.py b/tbc/pym/build_depgraph.py index 812e576..5d8b93e 100644 --- a/tbc/pym/build_depgraph.py +++ b/tbc/pym/build_depgraph.py @@ -19,7 +19,7 @@ def build_mydepgraph(settings, trees, mtimedb, myopts, myparams, myaction, myfil except portage.exception.PackageSetNotFound as e: root_config = trees[settings["ROOT"]]["root_config"] display_missing_pkg_set(root_config, e.value) - build_dict['type_fail'] = "depgraph fail" + build_dict['type_fail'] = "depgraph fail\n" build_dict['check_fail'] = True else: if not success: @@ -27,23 +27,23 @@ def build_mydepgraph(settings, trees, mtimedb, myopts, myparams, myaction, myfil repeat_times = 0 while repeat: if mydepgraph._dynamic_config._needed_p_mask_changes: - build_dict['type_fail'] = "Mask package or dep" + build_dict['type_fail'] = "Mask package or dep\n" build_dict['check_fail'] = True elif mydepgraph._dynamic_config._needed_use_config_changes: mydepgraph._display_autounmask() - build_dict['type_fail'] = "Need use change" + build_dict['type_fail'] = "Need use change\n" build_dict['check_fail'] = True elif mydepgraph._dynamic_config._slot_conflict_handler: - build_dict['type_fail'] = "Slot blocking" + build_dict['type_fail'] = "Slot blocking\n" build_dict['check_fail'] = True elif mydepgraph._dynamic_config._circular_deps_for_display: - build_dict['type_fail'] = "Circular Deps" + build_dict['type_fail'] = "Circular Deps\n" build_dict['check_fail'] = True elif mydepgraph._dynamic_config._unsolvable_blockers: - build_dict['type_fail'] = "Blocking packages" + build_dict['type_fail'] = "Blocking packages\n" build_dict['check_fail'] = True else: - build_dict['type_fail'] = "Dep calc fail" + build_dict['type_fail'] = "Dep calc fail\n" build_dict['check_fail'] = True mydepgraph.display_problems() if repeat_times is 2: diff --git a/tbc/pym/build_job.py b/tbc/pym/build_job.py index 8b8d393..37f2492 100644 --- a/tbc/pym/build_job.py +++ b/tbc/pym/build_job.py @@ -149,11 +149,11 @@ class build_job_action(object): log_msg = "build_job %s was not removed" % (build_dict['build_job_id'],) add_logs(self._session, log_msg, "info", self._config_id) print("qurery was not removed") - build_dict['type_fail'] = "Querey was not removed" + build_dict['type_fail'] = "Querey was not removed\n" build_dict['check_fail'] = True log_fail_queru(self._session, build_dict, settings) if build_fail is True: - build_dict['type_fail'] = "Emerge faild" + build_dict['type_fail'] = "Emerge faild\n" build_dict['check_fail'] = True log_msg = "Emerge faild!" add_logs(self._session, log_msg, "info", self._config_id) diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py index 9923b61..5c8ec74 100644 --- a/tbc/pym/build_log.py +++ b/tbc/pym/build_log.py @@ -29,7 +29,7 @@ from tbc.ConnectionManager import NewConnection from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \ get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \ add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \ - update_buildjobs_status, update_manifest_sql, add_repoman_qa + update_buildjobs_status, update_manifest_sql, add_repoman_qa, get_config_id_fqdn from sqlalchemy.orm import sessionmaker def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg): @@ -206,15 +206,20 @@ def get_buildlog_info(session, settings, pkg, build_dict): # Run repoman check_repoman() repoman_error_list = check_repoman(settings, myportdb, build_dict['cpv'], pkg.repo) + build_log_dict['rmqa'] = False + build_log_dict['fail'] = False if repoman_error_list: sum_build_log_list.append("1") # repoman = 1 + build_log_dict['rmqa'] = True if qa_error_list != []: sum_build_log_list.append("2") # qa = 2 + build_log_dict['rmqa'] = True else: qa_error_list = False error_search_line = "^ \\* ERROR: " for error_log_line in error_log_list: if re.search(error_search_line, error_log_line): + build_log_dict['fail'] = True for error_info in error_info_list: if re.search(error_info.ErrorSearch, error_log_line): sum_build_log_list.append(error_info.ErrorId) @@ -235,13 +240,15 @@ def get_emerge_info_id(settings, trees, session, config_id): return "\n".join(emerge_info_list) def add_buildlog_main(settings, pkg, trees): - tbc_settings_dict = read_config_settings() - config = tbc_settings_dict['tbc_config'] - hostname =tbc_settings_dict['hostname'] - host_config = hostname + "/" + config - Session = sessionmaker(bind=NewConnection(tbc_settings_dict)) + tbc_settings = read_config_settings() + Session = sessionmaker(bind=NewConnection(tbc_settings)) session = Session() - config_id = get_config_id(session, config, hostname) + hostname = tbc_settings['hostname'] + config_id = get_config_id_fqdn(session, host) + ConfigsMetaDataInfo = get_configmetadata_info(session, config_id) + ConfigInfo = get_config_info(session, config_id) + SetupInfo = get_setup_info(session, ConfigInfo.SetupId) + host_config = ConfigInfo.Hostname +"/" + SetupInfo.Setup if pkg.type_name == "binary": build_dict = None else: @@ -250,6 +257,7 @@ def add_buildlog_main(settings, pkg, trees): log_msg = "Package %s:%s is NOT logged." % (pkg.cpv, pkg.repo,) add_logs(session, log_msg, "info", config_id) session.close + Session.remove() return build_log_dict = {} build_log_dict = get_buildlog_info(session, settings, pkg, build_dict) @@ -279,6 +287,7 @@ def add_buildlog_main(settings, pkg, trees): add_logs(session, log_msg, "info", config_id) print("\n>>> Logging %s:%s\n" % (pkg.cpv, pkg.repo,)) session.close + Session.remove() def log_fail_queru(session, build_dict, settings): config_id = build_dict['config_id'] @@ -329,5 +338,6 @@ def log_fail_queru(session, build_dict, settings): build_log_dict['hilight_dict'] = {} settings2, trees, tmp = load_emerge_config() build_log_dict['emerge_info'] = get_emerge_info_id(settings2, trees, session, config_id) + build_log_dict['others'] = True log_id = add_new_buildlog(session, build_dict, build_log_dict) del_old_build_jobs(session, build_dict['build_job_id']) diff --git a/tbc/pym/check_setup.py b/tbc/pym/check_setup.py index 7367125..a5f53e7 100644 --- a/tbc/pym/check_setup.py +++ b/tbc/pym/check_setup.py @@ -11,17 +11,18 @@ from tbc.text import get_file_text from tbc.sqlquerys import get_config_all_info, add_logs, get_configmetadata_info, get_setup_info from tbc.sync import git_pull -def check_make_conf(session, config_id, tbc_settings_dict): +def check_make_conf(session, config_id): log_msg = "Checking configs for changes and errors" add_logs(session, log_msg, "info", config_id) - git_repo = tbc_settings_dict['tbc_gitrepopath'] + "/" + HostConfigsMetaDataInfo = get_configmetadata_info(session, config_id) + git_repo = HostConfigsMetaDataInfo.RepoPath + "/" git_pull(session, git_repo, config_id) configsDict = {} for ConfigInfo in get_config_all_info(session): attDict={} # Set the config dir SetupInfo = get_setup_info(session, ConfigInfo.ConfigId) - check_config_dir = tbc_settings_dict['tbc_gitrepopath'] + "/" + ConfigInfo.Hostname +"/" + SetupInfo.Setup + "/" + check_config_dir = git_repo + ConfigInfo.Hostname +"/" + SetupInfo.Setup + "/" make_conf_file = check_config_dir + "etc/portage/make.conf" ConfigsMetaDataInfo = get_configmetadata_info(session, ConfigInfo.ConfigId) # Check if we can take a checksum on it. @@ -51,8 +52,9 @@ def check_make_conf(session, config_id, tbc_settings_dict): log_msg = "Checking configs for changes and errors ... Done" add_logs(session, log_msg, "info", config_id) -def check_make_conf_guest(session, tbc_settings_dict, config_id): - git_repo = tbc_settings_dict['tbc_gitrepopath'] + "/" +def check_configure_guest(session, config_id): + GuestConfigsMetaDataInfo = get_configmetadata_info(session, config_id) + git_repo = GuestConfigsMetaDataInfo.RepoPath + "/" git_pull(session, git_repo, config_id) make_conf_file = "/etc/portage/make.conf" # Check if we can open the file and close it @@ -66,14 +68,6 @@ def check_make_conf_guest(session, tbc_settings_dict, config_id): # With errors we return false except Exception as e: return False - ConfigsMetaDataInfo = get_configmetadata_info(session, config_id) - print('make_conf_checksum_tree', make_conf_checksum_tree) - print('make_conf_checksum_db', ConfigsMetaDataInfo.Checksum) - if make_conf_checksum_tree != ConfigsMetaDataInfo.Checksum: + if make_conf_checksum_tree != GuestConfigsMetaDataInfo.Checksum: return False return True - -def check_configure_guest(session, tbc_settings_dict, config_id): - pass_make_conf = check_make_conf_guest(session, tbc_settings_dict, config_id) - print(pass_make_conf) - return pass_make_conf diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py index 76ff33e..05fccae 100644 --- a/tbc/pym/db_mapping.py +++ b/tbc/pym/db_mapping.py @@ -175,6 +175,8 @@ class BuildLogs(Base): BuildLogId = Column('build_log_id', Integer, primary_key=True) EbuildId = Column('ebuild_id', Integer, ForeignKey('ebuilds.ebuild_id')) Fail = Column('fail', Boolean, default=False) + RmQa = Column('rmqa', Boolean, default=False) + Others = Column('others', Boolean, default=False) SummeryText = Column('summery_text', Text) LogHash = Column('log_hash', String(100)) BugId = Column('bug_id', Integer, default=0) @@ -216,17 +218,11 @@ class BuildLogsUse(Base): Status = Column('status', Boolean, default=False) __tablename__ = 'build_logs_use' -class BuildLogsQA(Base): +class BuildLogsRepomanQa(Base): Id = Column('id', Integer, primary_key=True) BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id')) SummeryText = Column('summery_text', Text) - __tablename__ = 'build_logs_qa' - -class BuildLogsRepoman(Base): - Id = Column('id', Integer, primary_key=True) - BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id')) - SummeryText = Column('summery_text', Text) - __tablename__ = 'build_logs_repoman' + __tablename__ = 'build_logs_repoman_qa' class ErrorsInfo(Base): ErrorId = Column('error_id', Integer, primary_key=True) diff --git a/tbc/pym/package.py b/tbc/pym/package.py index 3e87aa6..440d141 100644 --- a/tbc/pym/package.py +++ b/tbc/pym/package.py @@ -18,36 +18,35 @@ from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \ class tbc_package(object): - def __init__(self, session, mysettings, myportdb, config_id, tbc_settings_dict): + def __init__(self, session, mysettings, myportdb, config_id): self._session = session self._mysettings = mysettings self._myportdb = myportdb self._config_id = config_id - self._tbc_settings_dict = tbc_settings_dict - def change_config(self, host_config): + def change_config(self, host_config, repopath): # Change config_root config_setup = table config - my_new_setup = tbc_settings_dict['tbc_gitrepopath'] + "/" + host_config + "/" + my_new_setup = repopath + "/" + host_config + "/" mysettings_setup = portage.config(config_root = my_new_setup) return mysettings_setup def config_match_ebuild(self, cp, config_list): - config_cpv_listDict ={} + config_cpv_dict ={} if config_list == []: - return config_cpv_listDict + return config_cpv_dict for config_id in config_list: ConfigInfo = get_config_info(self._session, config_id) ConfigsMetaData = get_configmetadata_info(self._session, config_id) - if ConfigsMetaData.Auto and ConfigsMetaData.Active and ConfigsMetaData.Status != 'Stopped': + if ConfigsMetaData.Auto and ConfigsMetaData.Active and ConfigsMetaData.Status != 'Stopped' and not ConfigInfo.SetupId in config_cpv_dict: SetupInfo = get_setup_info(self._session, config_id) - mysettings_setup = self.change_config(ConfigInfo.Hostname + "/" + SetupInfo.Setup) + mysettings_setup = self.change_config(ConfigInfo.Hostname + "/" + SetupInfo.Setup, ConfigsMetaData.RepoPath) myportdb_setup = portage.portdbapi(mysettings=mysettings_setup) # Get the latest cpv from portage with the config that we can build build_cpv = myportdb_setup.xmatch('bestmatch-visible', cp) # Check if could get cpv from portage and add it to the config_cpv_listDict. - if build_cpv != "" and not ConfigInfo.SetupId in config_cpv_listDict: + if build_cpv != "": # Get the iuse and use flags for that config/setup and cpv init_useflags = tbc_use_flags(mysettings_setup, myportdb_setup, build_cpv) @@ -61,19 +60,19 @@ class tbc_package(object): attDict['cpv'] = build_cpv attDict['useflags'] = final_use_list attDict['iuse'] = iuse_flags_list2 - config_cpv_listDict[ConfigInfo.SetupId] = attDict + config_cpv_dict[ConfigInfo.SetupId] = attDict # Clean some cache myportdb_setup.close_caches() portage.portdbapi.portdbapi_instances.remove(myportdb_setup) - return config_cpv_listDict + return config_cpv_dict def get_ebuild_metadata(self, cpv, repo): # Get the auxdbkeys infos for the ebuild try: ebuild_auxdb_list = self._myportdb.aux_get(cpv, portage.auxdbkeys, myrepo=repo) except: - ebuild_auxdb_list = None + ebuild_auxdb_list = False else: for i in range(len(ebuild_auxdb_list)): if ebuild_auxdb_list[i] == '': @@ -106,7 +105,7 @@ class tbc_package(object): # if there some error to get the metadata we add rubish to the # ebuild_version_metadata_tree and set ebuild_version_checksum_tree to 0 # so it can be updated next time we update the db - if ebuild_version_metadata_tree is None: + if ebuild_version_metadata_tree: log_msg = " QA: %s have broken metadata on repo %s" % (cpv, repo) add_logs(self._session, log_msg, "info", self._config_id) ebuild_version_metadata_tree = ['','','','','','','','','','','','','','','','','','','','','','','','',''] @@ -183,7 +182,7 @@ class tbc_package(object): log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml") add_logs(self._session, log_msg, "qa", self._config_id) attDict['metadata_xml_email'] = False - attDict['metadata_xml_descriptions'] = pkg_md.descriptions()[0] + attDict['metadata_xml_descriptions'] = pkg_md.descriptions() attDict['metadata_xml_checksum'] = portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0] #attDict['metadata_xml_text'] = metadata_xml_text_tree package_metadataDict[package_id] = attDict @@ -282,12 +281,12 @@ class tbc_package(object): new_ebuild_id_list = [] old_ebuild_id_list = [] for cpv in sorted(ebuild_list_tree): - repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo) - if repoman_fail: - log_msg = "Repoman %s:%s ... Fail." % (cpv, repo) - add_logs(self._session, log_msg, "error", self._config_id) packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo) - + if packageDict[cpv]['checksum'] == "0": + repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo) + if repoman_fail: + log_msg = "Repoman %s:%s ... Fail." % (cpv, repo) + add_logs(self._session, log_msg, "error", self._config_id) self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree) log_msg = "C %s:%s ... Done." % (cp, repo) add_logs(self._session, log_msg, "info", self._config_id) @@ -348,7 +347,8 @@ class tbc_package(object): # Get the checksum of the ebuild in tree and db ebuild_version_checksum_tree = packageDict[cpv]['checksum'] - checksums_db, fail= get_ebuild_checksums(self._session, package_id, ebuild_version_tree) + checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree) + # check if we have dupes of the checksum from db if checksums_db is None: ebuild_version_manifest_checksum_db = None @@ -365,11 +365,12 @@ class tbc_package(object): else: ebuild_version_manifest_checksum_db = checksums_db - if ebuild_version_manifest_checksum_db is None or ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db: + # Check with repoman + if (ebuild_version_manifest_checksum_db is None or ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db) and ebuild_version_checksum_tree != "0": repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo) if repoman_fail: log_msg = "Repoman %s:%s ... Fail." % (cpv, repo) - add_logs(self._session, log_msg, "info", self._config_id) + add_logs(self._session, log_msg, "error", self._config_id) # Check if the checksum have change if ebuild_version_manifest_checksum_db is None: diff --git a/tbc/pym/qachecks.py b/tbc/pym/qachecks.py index 36d2a31..32d27b3 100644 --- a/tbc/pym/qachecks.py +++ b/tbc/pym/qachecks.py @@ -57,7 +57,7 @@ def digestcheck(mysettings, pkgdir): except FileNotFound as e: return ("\n!!! A file listed in the Manifest could not be found: %s\n") % str(e) except DigestException as e: - return ("!!! Digest verification failed: %s\nReason: %s\nGot: %s\nExpected: %s") \ + return ("!!! Digest verification failed: %s\nReason: %s\nGot: %s\nExpected: %s\n") \ % (e.value[0], e.value[1], e.value[2], e.value[3]) if mf.thin or mf.allow_missing: # In this case we ignore any missing digests that @@ -82,7 +82,7 @@ def digestcheck(mysettings, pkgdir): except UnicodeDecodeError: parent = _unicode_decode(parent, encoding=_encodings['fs'], errors='replace') - return ("!!! Path contains invalid character(s) for encoding '%s': '%s'") % (_encodings['fs'], parent) + return ("!!! Path contains invalid character(s) for encoding '%s': '%s'\n") % (_encodings['fs'], parent) for d in dirs: d_bytes = d try: @@ -91,7 +91,7 @@ def digestcheck(mysettings, pkgdir): except UnicodeDecodeError: d = _unicode_decode(d, encoding=_encodings['fs'], errors='replace') - return ("!!! Path contains invalid character(s) for encoding '%s': '%s'") % (_encodings['fs'], os.path.join(parent, d)) + return ("!!! Path contains invalid character(s) for encoding '%s': '%s'\n") % (_encodings['fs'], os.path.join(parent, d)) if d.startswith(".") or d == "CVS": dirs.remove(d_bytes) for f in files: @@ -104,7 +104,7 @@ def digestcheck(mysettings, pkgdir): if f.startswith("."): continue f = os.path.join(parent, f)[len(filesdir) + 1:] - return ("!!! File name contains invalid character(s) for encoding '%s': '%s'") % (_encodings['fs'], f) + return ("!!! File name contains invalid character(s) for encoding '%s': '%s'\n") % (_encodings['fs'], f) if f.startswith("."): continue f = os.path.join(parent, f)[len(filesdir) + 1:] @@ -120,7 +120,7 @@ def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list ebuild_version = portage.versions.cpv_getversion(cpv) package = portage.versions.cpv_getkey(cpv).split("/")[1] if my_manifest.findFile(package + "-" + ebuild_version + ".ebuild") is None: - return "Ebuild file not found." + return "Ebuild file not found.\n" tree = portdb.getRepositoryPath(repo) cpv_fetchmap = portdb.getFetchMap(cpv, useflags=build_use_flags_list, mytree=tree) mysettings.unlock() @@ -128,17 +128,17 @@ def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list portage.fetch(cpv_fetchmap, mysettings, listonly=0, fetchonly=0, locks_in_subdir='.locks', use_locks=1, try_mirrors=1) except: mysettings.lock() - return "Can't fetch the file." + return "Can't fetch the file.\n" finally: mysettings.lock() try: my_manifest.checkCpvHashes(cpv, checkDistfiles=True, onlyDistfiles=False, checkMiscfiles=True) except: - return "Can't fetch the file or the hash failed." + return "Can't fetch the file or the hash failed.\n" try: portdb.fetch_check(cpv, useflags=build_use_flags_list, mysettings=mysettings, all=False) except: - return "Fetch check failed." + return "Fetch check failed.\n" return def check_repoman(mysettings, myportdb, cpv, repo): @@ -167,7 +167,7 @@ def check_repoman(mysettings, myportdb, cpv, repo): mode = 'r', encoding = _encodings['repo.content']) try: for check_name, e in run_checks(f, pkg): - fails.append(check_name + ": " + e) + fails.append(check_name + ": " + e + "\n") finally: f.close() except UnicodeDecodeError: diff --git a/tbc/pym/readconf.py b/tbc/pym/readconf.py index 39131db..b4bad11 100644 --- a/tbc/pym/readconf.py +++ b/tbc/pym/readconf.py @@ -6,53 +6,34 @@ import sys import re from socket import getfqdn -class get_conf_settings(object): -# open the /etc/tbc/tbc.conf file and get the needed -# settings for tbc - def __init__(self): - self.configfile = "/etc/tbc/tbc.conf" - - def read_tbc_settings_all(self): - # It will return a dict with options from the configfile - try: - open_conffile = open(self.configfile, 'r') - except: - sys.exit("Fail to open config file:" + self.configfile) - textlines = open_conffile.readlines() - for line in textlines: - element = line.split('=') - if element[0] == 'SQLBACKEND': # Databas backend - get_sql_backend = element[1] - if element[0] == 'SQLDB': # Database - get_sql_db = element[1] - if element[0] == 'SQLHOST': # Host - get_sql_host = element[1] - if element[0] == 'SQLUSER': # User - get_sql_user = element[1] - if element[0] == 'SQLPASSWD': # Password - get_sql_passwd = element[1] - # Buildhost root (dir for host/setup on host) - if element[0] == 'TBCREPOPATH': - get_tbc_gitrepopath = element[1] - # Buildhost setup (host/setup on guest) - if element[0] == 'TBCCONFIG': - get_tbc_config = element[1] - # if element[0] == 'LOGFILE': - # get_tbc_logfile = element[1] - open_conffile.close() - - tbc_settings_dict = {} - tbc_settings_dict['sql_backend'] = get_sql_backend.rstrip('\n') - tbc_settings_dict['sql_db'] = get_sql_db.rstrip('\n') - tbc_settings_dict['sql_host'] = get_sql_host.rstrip('\n') - tbc_settings_dict['sql_user'] = get_sql_user.rstrip('\n') - tbc_settings_dict['sql_passwd'] = get_sql_passwd.rstrip('\n') - tbc_settings_dict['tbc_gitrepopath'] = get_tbc_gitrepopath.rstrip('\n') - tbc_settings_dict['tbc_config'] = get_tbc_config.rstrip('\n') - tbc_settings_dict['hostname'] = getfqdn() - # tbc_settings_dict['tbc_logfile'] = get_tbc_logfile.rstrip('\n') - return tbc_settings_dict +configfile = "/etc/tbc/tbc.conf" def read_config_settings(): - reader = get_conf_settings() - return reader.read_tbc_settings_all() +# It will return a dict with options from the configfile + try: + open_conffile = open(configfile, 'r') + except: + sys.exit("Fail to open config file:" + configfile) + textlines = open_conffile.readlines() + for line in textlines: + element = line.split('=') + if element[0] == 'SQLBACKEND': # Databas backend + get_sql_backend = element[1] + if element[0] == 'SQLDB': # Database + get_sql_db = element[1] + if element[0] == 'SQLHOST': # Host + get_sql_host = element[1] + if element[0] == 'SQLUSER': # User + get_sql_user = element[1] + if element[0] == 'SQLPASSWD': # Password + get_sql_passwd = element[1] + open_conffile.close() + + tbc_settings = {} + tbc_settings['sql_backend'] = get_sql_backend.rstrip('\n') + tbc_settings['sql_db'] = get_sql_db.rstrip('\n') + tbc_settings['sql_host'] = get_sql_host.rstrip('\n') + tbc_settings['sql_user'] = get_sql_user.rstrip('\n') + tbc_settings['sql_passwd'] = get_sql_passwd.rstrip('\n') + tbc_settings['hostname'] = getfqdn() + return tbc_settings diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py index b897f16..51a266c 100644 --- a/tbc/pym/sqlquerys.py +++ b/tbc/pym/sqlquerys.py @@ -3,6 +3,7 @@ from __future__ import print_function import datetime +import sys from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Packages, Ebuilds, Repos, Categories, \ Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs, BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \ HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \ @@ -17,6 +18,10 @@ def get_config_id(session, setup, host): ConfigInfo = session.query(Configs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Hostname = host).one() return ConfigInfo.ConfigId +def get_config_id_fqdn(session, host): + ConfigInfo = session.query(Configs).filter_by(Hostname = host).one() + return ConfigInfo.ConfigId + def add_logs(session, log_msg, log_type, config_id): Add_Log = Logs(ConfigId = config_id, LogType = log_type, Msg = log_msg) session.add(Add_Log) @@ -224,10 +229,7 @@ def add_new_buildlog(session, build_dict, build_log_dict): return None, False def build_log_id_no_match(build_dict, build_log_dict): - if build_log_dict['summary_error_list'] == []: - NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = False, SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash']) - else: - NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = True, SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash']) + NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = build_log_dict['fail'], RmQa = build_log_dict['rmqa'], Others = build_log_dict['others'], SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash']) session.add(NewBuildLog) session.flush() build_log_id = NewBuildLog.BuildLogId @@ -274,16 +276,14 @@ def add_new_buildlog(session, build_dict, build_log_dict): def add_repoman_qa(session, build_log_dict, log_id): repoman_error = "" qa_error = "" - if build_log_dict['qa_error_list']: - for qa_text in build_log_dict['qa_error_list']: - qa_error = qa_error + build_log_dict['qa_error_list'] - NewBuildLogQA = BuildLogsQA(BuildLogId = log_id, SummeryText = qa_error) - session.add(NewBuildLogQA) - session.commit() if build_log_dict['repoman_error_list']: for repoman_text in build_log_dict['repoman_error_list']: repoman_error = repoman_error + repoman_text - NewBuildLogRepoman = BuildLogsRepoman(BuildLogId = log_id, SummeryText = repoman_error) + if build_log_dict['qa_error_list']: + for qa_text in build_log_dict['qa_error_list']: + qa_error = qa_error + qa_text + repoman_error = repoman_error + qa_error + NewBuildLogRepoman = BuildLogsRepomanQa(BuildLogId = log_id, SummeryText = repoman_error) session.add(NewBuildLogRepoman) session.commit() @@ -391,12 +391,15 @@ def add_new_ebuild_sql(session, packageDict): except (MultipleResultsFound) as e: for x in session.query(Ebuilds).filter_by(Version = v['ebuild_version']).filter_by(Checksum = v['checksum']).\ filter_by(PackageId = v['package_id']).filter_by(Active = True).all(): - print(x.EbuildId) + x.Checksum = 0 + x.Active = False + session.commit() + try: + EbuildInfo = session.query(Ebuilds).filter_by(Version = v['ebuild_version']).filter_by(Checksum = v['checksum']).\ + filter_by(PackageId = v['package_id']).filter_by(Active = True).one() + except (MultipleResultsFound) as e: # FIXME - #x.Checksum = 0 - #x.Active = False - #session.commit() - sys.exit() + sys.exit() session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, Revision = v['ebuild_version_revision_tree'], Descriptions = v['ebuild_version_descriptions_tree'])) session.commit() ebuild_id_list.append(EbuildInfo.EbuildId) diff --git a/tbc/pym/sync.py b/tbc/pym/sync.py index e9a2aca..c7d2ebb 100644 --- a/tbc/pym/sync.py +++ b/tbc/pym/sync.py @@ -10,7 +10,7 @@ import time import re import git -from tbc.sqlquerys import get_config_id, add_logs, get_config_all_info, get_configmetadata_info +from tbc.sqlquerys import get_config_id_fqdn, add_logs, get_config_all_info, get_configmetadata_info from tbc.readconf import read_config_settings def git_repos_list(myportdb): @@ -34,12 +34,14 @@ def git_merge(repo, info): repo.git.merge(info.commit) def git_sync_main(session): - tbc_settings_dict = read_config_settings() - _hostname = tbc_settings_dict['hostname'] - _config = tbc_settings_dict['tbc_config'] - config_id = get_config_id(session, _config, _hostname) - host_config = _hostname +"/" + _config - default_config_root = tbc_settings_dict['tbc_gitrepopath'] + "/" + host_config + "/" + tbc_settings = read_config_settings() + hostname = tbc_settings_dict['hostname'] + config_id = get_config_id_fqdn(session, host) + ConfigsMetaDataInfo = get_configmetadata_info(session, config_id) + ConfigInfo = get_config_info(session, config_id) + SetupInfo = get_setup_info(session, ConfigInfo.SetupId) + host_config = ConfigInfo.Hostname +"/" + SetupInfo.Setup + default_config_root = ConfigsMetaDataInfo.RepoPath + "/" + host_config + "/" mysettings = portage.config(config_root = default_config_root) myportdb = portage.portdbapi(mysettings=mysettings) GuestBusy = True @@ -53,12 +55,12 @@ def git_sync_main(session): while GuestBusy: Status_list = [] for guest_id in guestid_list: - ConfigMetadata = get_configmetadata_info(session, guest_id) - Status_list.append(ConfigMetadata.Status) + ConfigMetadataGuest = get_configmetadata_info(session, guest_id) + Status_list.append(ConfigMetadataGuest.Status) if not 'Runing' in Status_list: GuestBusy = False else: - time.sleep(30) + time.sleep(60) #remove the needed base profile clone try: os.remove(mysettings['PORTDIR'] + "/profiles/config/parent") @@ -76,6 +78,7 @@ def git_sync_main(session): info_list, repouptodate = git_fetch(repo) if not repouptodate: cp_list = [] + # FiXME We still miss files to update for diff_line in repo.git.diff('HEAD^').splitlines(): if re.search("^diff --git.*/Manifest", diff_line): diff_line2 = re.split(' b/', re.sub('diff --git', '', diff_line)) diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py index 321c34e..6ccf0b6 100644 --- a/tbc/pym/updatedb.py +++ b/tbc/pym/updatedb.py @@ -14,18 +14,21 @@ from tbc.sqlquerys import add_logs, get_package_info, update_repo_db, \ get_config_info from tbc.check_setup import check_make_conf from tbc.package import tbc_package -# Get the options from the config file set in tbc.readconf +# Get the options from the config file tbc.conf from tbc.readconf import read_config_settings -def init_portage_settings(session, config_id, tbc_settings_dict): +def init_portage_settings(session, config_id): # check config setup - check_make_conf(session, config_id, tbc_settings_dict) + check_make_conf(session, config_id) log_msg = "Check configs done" add_logs(session, log_msg, "info", config_id) - - # Get default config from the configs table and default_config=1 - host_config = tbc_settings_dict['hostname'] +"/" + tbc_settings_dict['tbc_config'] - default_config_root = tbc_settings_dict['tbc_gitrepopath'] + "/" + host_config + "/" + + # setup default root + ConfigsMetaDataInfo = get_configmetadata_info(session, config_id) + ConfigInfo = get_config_info(session, config_id) + SetupInfo = get_setup_info(session, ConfigInfo.SetupId) + host_config = ConfigInfo.Hostname +"/" + SetupInfo.Setup + default_config_root = ConfigsMetaDataInfo.RepoPath + "/" + host_config + "/" # Set config_root (PORTAGE_CONFIGROOT) to default_config_root mysettings = portage.config(config_root = default_config_root) @@ -33,11 +36,11 @@ def init_portage_settings(session, config_id, tbc_settings_dict): add_logs(session, log_msg, "info", config_id) return mysettings -def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings_dict, config_id): - session_factory = sessionmaker(bind=NewConnection(tbc_settings_dict)) +def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings, config_id): + session_factory = sessionmaker(bind=NewConnection(tbc_settings)) Session = scoped_session(session_factory) session2 = Session() - init_package = tbc_package(session2, mysettings, myportdb, config_id, tbc_settings_dict) + init_package = tbc_package(session2, mysettings, myportdb, config_id) # split the cp to categories and package element = cp.split('/') @@ -55,9 +58,10 @@ def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings_dict, config else: # Add new package with ebuilds init_package.add_new_package_db(cp, repo) + session2.close Session.remove() -def update_cpv_db(session, config_id, tbc_settings_dict): +def update_cpv_db(session, config_id, tbc_settings): GuestBusy = True log_msg = "Waiting for Guest to be idel" add_logs(session, log_msg, "info", config_id) @@ -79,7 +83,7 @@ def update_cpv_db(session, config_id, tbc_settings_dict): new_build_jobs_list = [] # Setup settings, portdb and pool - mysettings = init_portage_settings(session, config_id, tbc_settings_dict) + mysettings = init_portage_settings(session, config_id) myportdb = portage.portdbapi(mysettings=mysettings) # Use all cores when multiprocessing @@ -107,9 +111,9 @@ def update_cpv_db(session, config_id, tbc_settings_dict): # Run the update package for all package in the list and in a multiprocessing pool for cp in sorted(package_list_tree): - pool.apply_async(update_cpv_db_pool, (mysettings, myportdb, cp, repo, zobcs_settings_dict, config_id,)) + # pool.apply_async(update_cpv_db_pool, (mysettings, myportdb, cp, repo, tbc_settings, config_id,)) # use this when debuging - #update_cpv_db_pool(mysettings, myportdb, cp, repo, zobcs_settings_dict, config_id) + update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings, config_id) else: # Update needed repos and packages in the dict for repo, v in repo_cp_dict.items(): @@ -120,14 +124,14 @@ def update_cpv_db(session, config_id, tbc_settings_dict): # Run the update package for all package in the list and in a multiprocessing pool for cp in v['cp_list']: - pool.apply_async(update_cpv_db_pool, (mysettings, myportdb, cp, repo, zobcs_settings_dict, config_id,)) + # pool.apply_async(update_cpv_db_pool, (mysettings, myportdb, cp, repo, tbc_settings, config_id,)) # use this when debuging - #update_cpv_db_pool(mysettings, myportdb, cp, repo, zobcs_settings_dict, config_id) + update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings, config_id) #close and join the multiprocessing pools - pool.close() - pool.join() + # pool.close() + # pool.join() log_msg = "Checking categories, package and ebuilds ... done" add_logs(session, log_msg, "info", config_id) @@ -136,12 +140,12 @@ def update_db_main(session, repo_cp_dict, config_id): if repo_cp_dict == {}: return True # Logging - tbc_settings_dict = reader. read_config_settings() + tbc_settings = read_config_settings() log_msg = "Update db started." add_logs(session, log_msg, "info", config_id) # Update the cpv db - update_cpv_db(session, repo_cp_dict, config_id, tbc_settings_dict) + update_cpv_db(session, repo_cp_dict, config_id, tbc_settings) log_msg = "Update db ... Done." add_logs(session, log_msg, "info", config_id) return True
