commit: 6d8986da642b5d3b5670be40d8fe8883086c7678
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 21 21:15:49 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Feb 21 21:15:49 2024 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6d8986da
Move search pattern db to settings repository
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/steps/logs.py | 34 ++++---
py/log_parser.py | 206 ++++++++++++++++++++-------------------
2 files changed, 127 insertions(+), 113 deletions(-)
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 85017a5..50e3f3f 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -42,10 +42,13 @@ def PersOutputOfLogParser(rc, stdout, stderr):
}
for s in v['pattern_info']:
summary_log_dict[int(k)]['pattern_infos'].append({
- 'type' : s['type'],
+ 'search_type' :
s['search_type'],
'status' :
s['status'],
- 'id' : s['id'],
+ 'line' : s['line'],
'search_pattern' :
s['search_pattern'],
+ 'phase' :
s['phase'],
+ 'uuid' : s['uuid'],
+ 'description' :
s['description'],
})
build_summery_output['summary_log_dict'] = summary_log_dict
#FIXME: Handling of stderr output
@@ -118,14 +121,23 @@ class SetupParserBuildLoger(BuildStep):
@defer.inlineCallbacks
def run(self):
+ self.gentooci =
self.master.namedServices['services'].namedServices['gentooci']
self.aftersteps_list = []
log_cpv =
self.getProperty('log_build_data')[self.getProperty('log_cpv')]
build_log_file_compressed = log_cpv['full_logname'] + '.xz'
mastersrc_log = yield os.path.join(self.getProperty('logsdir'),
build_log_file_compressed)
log_py = 'log_parser.py'
- config_log_py = 'logparser.json'
mastersrc_py = yield os.path.join(self.master.basedir, log_py)
- mastersrc_config = yield os.path.join(self.master.basedir,
config_log_py)
+ repository_data = yield
self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty("project_data")['settings_repository_uuid'])
+ # Git clone settings repo
+
self.aftersteps_list.append(steps.GitLab(repourl=repository_data['url'],
+ name = 'RunGit',
+ descriptionDone=repository_data['name'],
+ mode=repository_data['mode'],
+ method=repository_data['method'],
+ submodules=True,
+ alwaysUseLatest=repository_data['alwaysuselatest']
+ ))
# Upload logfile to worker
self.aftersteps_list.append(steps.FileDownload(
mastersrc=mastersrc_log,
@@ -136,11 +148,6 @@ class SetupParserBuildLoger(BuildStep):
mastersrc=mastersrc_py,
workerdest=log_py
))
- # Upload log parser py config
- self.aftersteps_list.append(steps.FileDownload(
- mastersrc=mastersrc_config,
- workerdest=config_log_py
- ))
#Untar the log
shell_commad_list = []
shell_commad_list.append('xz')
@@ -159,8 +166,11 @@ class SetupParserBuildLoger(BuildStep):
command.append(log_cpv['full_logname'])
command.append('-u')
command.append(self.getProperty('project_data')['uuid'])
- command.append('-d')
- command.append(util.Secret("log_parser_database"))
+ command.append('-c')
+ #FIXME: set it by images/flavors
+ command.append('8')
+ #FIXME: debug
+ #command.append('-d')
self.aftersteps_list.append(steps.SetPropertyFromCommand(
name =
'RunBuildLogParser',
haltOnFailure =
True,
@@ -211,7 +221,7 @@ class MakeIssue(BuildStep):
for k, v in sorted(self.summary_log_dict.items()):
# get the issue error
for s in v['pattern_infos']:
- if s['type'] == self.error_dict['phase'] and s['status'] ==
'error':
+ if s['search_type'] == self.error_dict['phase'] and
s['status'] == 'error':
text_issue_list.append(v['text'])
# add the issue error
self.error_dict['cpv'] = self.getProperty('log_cpv')
diff --git a/py/log_parser.py b/py/log_parser.py
index eb081a9..e3725bb 100644
--- a/py/log_parser.py
+++ b/py/log_parser.py
@@ -7,113 +7,113 @@ import re
import io
import json
import os
-from sqlalchemy.ext.declarative import declarative_base
-import sqlalchemy as sa
import argparse
-Base = declarative_base()
-
-class ProjectsPattern(Base):
- __tablename__ = "projects_pattern"
- id = sa.Column(sa.Integer, primary_key=True)
- project_uuid = sa.Column(sa.String(36), nullable=False)
- search = sa.Column(sa.String(50), nullable=False)
- start = sa.Column(sa.Integer, default=0)
- end = sa.Column(sa.Integer, default=0)
- status = sa.Column(sa.Enum('info', 'warning', 'ignore', 'error'),
default='info')
- type = sa.Column(sa.Enum('info', 'qa', 'compile', 'configure', 'install',
'postinst', 'prepare', 'pretend', 'setup', 'test', 'unpack', 'ignore',
'issues', 'misc', 'elog'), default='info')
- search_type = sa.Column(sa.Enum('in', 'startswith', 'endswith', 'search'),
default='in')
+def getJsonFromFile(path, phase):
+ with open(path) as f:
+ try:
+ JsonList = json.load(f)
+ except json.decoder.JSONDecodeError as e:
+ print(f"{e} in file: {path}")
+ return []
+ return JsonList[phase]
-def get_pattern_dict(project_pattern):
+def get_pattern_dict(catchissue, i, uuid):
+ #print(catchissue)
patten_dict = {}
- patten_dict['id'] = project_pattern.id
- patten_dict['project_uuid'] = project_pattern.project_uuid
- patten_dict['search'] = project_pattern.search
- patten_dict['status'] = project_pattern.status
- patten_dict['type'] = project_pattern.type
+ patten_dict['line'] = i
+ patten_dict['uuid'] = uuid
+ patten_dict['string'] = catchissue[0]
+ patten_dict['start'] = catchissue[1]
+ patten_dict['end'] = catchissue[2]
+ patten_dict['status'] = catchissue[3]
+ patten_dict['search_type'] = catchissue[4]
+ patten_dict['url'] = catchissue[5]
+ patten_dict['description'] = catchissue[6]
return patten_dict
-def addPatternToList(Session, log_search_pattern, uuid):
- for project_pattern in
Session.query(ProjectsPattern).filter_by(project_uuid=uuid).all():
- # check if the search pattern is vaild
- project_pattern_search = project_pattern.search
- try:
- re.compile(project_pattern_search)
- except re.error:
- print("Non valid regex pattern")
- print(project_pattern.search)
- print(project_pattern.id)
+def addPatternToList(phaseList, log_search_patterns, uuid):
+ for phase in phaseList:
+ if uuid == '00000000-0000-0000-000000000000':
+ path = 'LogPattern'
else:
- if project_pattern.type == 'ignore':
-
log_search_pattern['ignore'].append(get_pattern_dict(project_pattern))
- if project_pattern.type == 'test':
-
log_search_pattern['test'].append(get_pattern_dict(project_pattern))
+ path = os.path.join('Project', uuid, 'LogPattern')
+ CatchIssueFile = os.path.join(path, 'CatchIssues' + phase + '.json')
+ CatchIssueList = getJsonFromFile(CatchIssueFile, phase)
+ i = 3
+ for catchissue in CatchIssueList:
+ search_pattern = get_pattern_dict(catchissue, i, uuid)
+ try:
+ re.compile(search_pattern['string'])
+ except re.error:
+ print(f"Non valid regex pattern in line:
{str(search_pattern['line'])} String: {search_pattern['string']} Project:
{search_pattern['uuid']} Phase: {phase}")
else:
-
log_search_pattern['default'].append(get_pattern_dict(project_pattern))
- return log_search_pattern
+ log_search_patterns[phase].append(search_pattern)
+ i = i + 1
+ return log_search_patterns
-def get_log_search_pattern(Session, uuid, default_uuid):
+def get_log_search_patterns(uuid):
+ path = os.path.join('LogPattern', 'PhaseList.json')
+ PhaseList = getJsonFromFile(path,'PhaseList')
# get pattern from the projects and add that to log_search_pattern
- log_search_pattern = {}
- log_search_pattern['ignore'] = []
- log_search_pattern['default'] = []
- log_search_pattern['test'] = []
- log_search_pattern = addPatternToList(Session, log_search_pattern, uuid)
- log_search_pattern = addPatternToList(Session, log_search_pattern,
default_uuid)
- return log_search_pattern
+ log_search_patterns = {}
+ for phase in PhaseList:
+ log_search_patterns[phase] = []
+ uuid_default = '00000000-0000-0000-000000000000'
+ log_search_patterns = addPatternToList(PhaseList, log_search_patterns,
uuid_default)
+ #log_search_pattern = addPatternToList(PhaseList, log_search_pattern, uuid)
+ return log_search_patterns
-def get_search_pattern_match(log_search_pattern, text_line):
- match_list = []
- for search_pattern in log_search_pattern:
- if re.search(search_pattern['search'], text_line):
- match_list.append(search_pattern)
- return match_list
+def get_search_pattern_match(search_pattern, text_line):
+ #print(f"Text: {text_line}")
+ if search_pattern['search_type'] == 'search':
+ if re.search(search_pattern['string'], text_line):
+ #print(f"Match string: {search_pattern['string']} Type:
{search_pattern['search_type']}")
+ return True
+ elif search_pattern['search_type'] == 'startswith':
+ if text_line.startswith(search_pattern['string']):
+ #print(f"Match string: {search_pattern['string']} Type:
{search_pattern['search_type']}")
+ return True
+ elif search_pattern['search_type'] == 'endswith':
+ if text_line.endswith(search_pattern['string']):
+ #print(f"Match string: {search_pattern['string']} Type:
{search_pattern['search_type']}")
+ return True
+ elif search_pattern['search_type'] == 'in':
+ if search_pattern['string'] in text_line:
+ #print(f"Match string: {search_pattern['string']} Type:
{search_pattern['search_type']}")
+ return True
+ else:
+ return False
-def search_buildlog(log_search_pattern, text_line, index):
+def search_buildlog(log_search_patterns, text_line, index):
summary = {}
- #FIXME: add check for test
- # don't log ignore lines
- if get_search_pattern_match(log_search_pattern['ignore'], text_line) != []:
- return False
- # search default pattern
summary[index] = {
'text' : text_line,
'pattern_info' : [],
}
- search_pattern_match_list =
get_search_pattern_match(log_search_pattern['default'], text_line)
- if search_pattern_match_list != []:
- for search_pattern_match in search_pattern_match_list:
- summary[index]['pattern_info'].append({
- 'type' :
search_pattern_match['type'],
- 'status' :
search_pattern_match['status'],
- 'id' : search_pattern_match['id'],
- 'search_pattern' :
search_pattern_match['search'],
- })
- # we add all line that start with ' * ' or '>>>' as info
- if text_line.startswith(' * ') or text_line.startswith('>>>'):
- summary[index]['pattern_info'].append({
- 'type' : 'info',
- 'status' : 'info',
- 'id' : 0,
- 'search_pattern' : 'auto',
- })
+ for phase, search_patterns in log_search_patterns.items():
+ for search_pattern in search_patterns:
+ match = get_search_pattern_match(search_pattern, text_line)
+ if phase == 'Ignore' and match:
+ return False
+ elif phase != 'Ignore' and match:
+ summary[index]['pattern_info'].append({
+ 'search_type' : search_pattern['search_type'],
+ 'status' : search_pattern['status'],
+ 'line' : search_pattern['line'],
+ 'search_pattern' : search_pattern['string'],
+ 'phase' : phase,
+ 'uuid' : search_pattern['uuid'],
+ 'url' : search_pattern['url'],
+ 'description' : search_pattern['description'],
+ })
if summary[index]['pattern_info'] != []:
+ #print(f"summary: {summary}")
return summary
return False
-def getConfigSettings():
- #configpath = os.getcwd()
- with open('logparser.json') as f:
- config = json.load(f)
- return config
-
-def getDBSession(args):
- engine = sa.create_engine(args.database)
- Session = sa.orm.sessionmaker(bind = engine)
- return Session()
-
-def getMultiprocessingPool(config):
- return Pool(processes = int(config['core']))
+def getMultiprocessingPool(args):
+ return Pool(processes = int(args.cpu))
def getJsonResult(results):
for r in results:
@@ -128,14 +128,11 @@ def getJsonResult(results):
def runLogParser(args):
index = 1
logfile_text_dict = {}
- config = getConfigSettings()
- Session = getDBSession(args)
- #mp_pool = getMultiprocessingPool(config)
summary = {}
#NOTE: The patten is from
https://github.com/toralf/tinderbox/tree/master/data files.
- # Is stored in a db instead of files.
- log_search_pattern = get_log_search_pattern(Session, args.uuid,
config['default_uuid'])
- Session.close()
+ # Is stored in json files
+ # make dict with it
+ log_search_patterns = get_log_search_patterns(args.uuid)
# read the log file to dict
with open(args.file, encoding='utf8', errors='ignore') as f:
for text_line in f:
@@ -146,21 +143,28 @@ def runLogParser(args):
# index = index + 1
# run the search parse pattern on the text lines
#params = [(log_search_pattern, text, line_index,) for line_index, text in
logfile_text_dict.items()]
- with getMultiprocessingPool(config) as pool:
- results = list(pool.apply_async(search_buildlog,
args=(log_search_pattern, text, line_index,)) for line_index, text in
logfile_text_dict.items())
- #results = pool.starmap(search_buildlog, params)
- getJsonResult(results)
- pool.close()
- pool.join()
+ if not args.debug:
+ with getMultiprocessingPool(args) as pool:
+ results = list(pool.apply_async(search_buildlog,
args=(log_search_patterns, text, line_index,)) for line_index, text in
logfile_text_dict.items())
+ getJsonResult(results)
+ pool.close()
+ pool.join()
+ else:
+ results = []
+ for line_index, text in logfile_text_dict.items():
+ results.append(search_buildlog(log_search_pattern, text,
line_index))
+ #FIXME: Json output
+ #getJsonResult(results)
def main():
# get filename, project_uuid default_project_uuid
parser = argparse.ArgumentParser()
parser.add_argument("-f", "--file", required=True)
parser.add_argument("-u", "--uuid", required=True)
- parser.add_argument("-e", "--default-uuid", required=False)
- parser.add_argument("-c", "--cpu", required=False)
- parser.add_argument("-d", "--database", required=True)
+ parser.add_argument("-c", "--cpu", required=True)
+ #FIXME: add If args.debug .... wear is needed
+ parser.add_argument("-d", "--debug", action="store_true", required=False)
+
args = parser.parse_args()
runLogParser(args)
sys.exit()
