Hello community,
here is the log from the commit of package openSUSE-release-tools for
openSUSE:Factory checked in at 2018-08-31 10:41:43
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old)
and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openSUSE-release-tools"
Fri Aug 31 10:41:43 2018 rev:123 rq:630991 version:20180822.a9f1bc0
Changes:
--------
---
/work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes
2018-08-22 14:20:03.950310868 +0200
+++
/work/SRC/openSUSE:Factory/.openSUSE-release-tools.new/openSUSE-release-tools.changes
2018-08-31 10:43:22.283125874 +0200
@@ -1,0 +2,23 @@
+Wed Aug 22 22:38:19 UTC 2018 - [email protected]
+
+- Update to version 20180822.a9f1bc0:
+ * osclib/core: repository_path_expand(): skip adding duplicate path.
+
+-------------------------------------------------------------------
+Wed Aug 22 02:02:18 UTC 2018 - [email protected]
+
+- Update to version 20180821.fa39e68:
+ * StagingAPI: drop inferior expanded_repos() implementation for osclib.core.
+ * pkglistgen: utilize osclib.core.repository_path_expand().
+ * repo_checker: complete rework to handle arbitrary repos and maintenance.
+ * osclib/util: provide sha1_short() adapted from repo_checker.
+ * osclib/core: provide project_meta_revision() adapted from repo_checker.
+ * osclib/core: provide repository state and published functions.
+ * osclib/core: provide repository_path_search().
+ * osclib/core: provide repository_path_expand() adapted from StagingAPI.
+ * osclib/core: target_archs(): expose repository argument.
+ * osclib/conf: drop main-repo default for all projects.
+ * ReviewBot: utilize osclib.Cache for all bots by default.
+ * ReviewBot: utilize memoize cached config.
+
+-------------------------------------------------------------------
Old:
----
openSUSE-release-tools-20180820.d7d5724.obscpio
New:
----
openSUSE-release-tools-20180822.a9f1bc0.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ openSUSE-release-tools.spec ++++++
--- /var/tmp/diff_new_pack.nozftN/_old 2018-08-31 10:43:23.111126855 +0200
+++ /var/tmp/diff_new_pack.nozftN/_new 2018-08-31 10:43:23.115126860 +0200
@@ -20,7 +20,7 @@
%define source_dir openSUSE-release-tools
%define announcer_filename factory-package-news
Name: openSUSE-release-tools
-Version: 20180820.d7d5724
+Version: 20180822.a9f1bc0
Release: 0
Summary: Tools to aid in staging and release work for openSUSE/SUSE
License: GPL-2.0-or-later AND MIT
++++++ _servicedata ++++++
--- /var/tmp/diff_new_pack.nozftN/_old 2018-08-31 10:43:23.143126893 +0200
+++ /var/tmp/diff_new_pack.nozftN/_new 2018-08-31 10:43:23.143126893 +0200
@@ -1,6 +1,6 @@
<servicedata>
<service name="tar_scm">
<param
name="url">https://github.com/openSUSE/openSUSE-release-tools.git</param>
- <param
name="changesrevision">1fe03c16f0b836db70d0bd4145161a1199952a60</param>
+ <param
name="changesrevision">a8cfd74f1f7a4a1a38fd2530262d0b7dd06d8177</param>
</service>
</servicedata>
++++++ openSUSE-release-tools-20180820.d7d5724.obscpio ->
openSUSE-release-tools-20180822.a9f1bc0.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openSUSE-release-tools-20180820.d7d5724/ReviewBot.py
new/openSUSE-release-tools-20180822.a9f1bc0/ReviewBot.py
--- old/openSUSE-release-tools-20180820.d7d5724/ReviewBot.py 2018-08-21
04:03:44.000000000 +0200
+++ new/openSUSE-release-tools-20180822.a9f1bc0/ReviewBot.py 2018-08-23
00:33:11.000000000 +0200
@@ -26,6 +26,7 @@
import cmdln
from collections import namedtuple
from collections import OrderedDict
+from osclib.cache import Cache
from osclib.comments import CommentAPI
from osclib.conf import Config
from osclib.core import group_members
@@ -141,7 +142,7 @@
def staging_api(self, project):
if project not in self.staging_apis:
- Config(self.apiurl, project)
+ Config.get(self.apiurl, project)
self.staging_apis[project] = StagingAPI(self.apiurl, project)
return self.staging_apis[project]
@@ -680,6 +681,7 @@
class CommandLineInterface(cmdln.Cmdln):
def __init__(self, *args, **kwargs):
cmdln.Cmdln.__init__(self, args, kwargs)
+ Cache.init()
self.clazz = ReviewBot
def get_optparser(self):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20180820.d7d5724/osclib/conf.py
new/openSUSE-release-tools-20180822.a9f1bc0/osclib/conf.py
--- old/openSUSE-release-tools-20180820.d7d5724/osclib/conf.py 2018-08-21
04:03:44.000000000 +0200
+++ new/openSUSE-release-tools-20180822.a9f1bc0/osclib/conf.py 2018-08-23
00:33:11.000000000 +0200
@@ -58,6 +58,7 @@
'review-team': 'opensuse-review-team',
'legal-review-group': 'legal-auto',
'repo-checker': 'repo-checker',
+ 'repo_checker-no-filter': 'True',
'pkglistgen-product-family-include': 'openSUSE:Leap:N',
'mail-list': '[email protected]',
'mail-maintainer': 'Dominique Leuenberger <[email protected]>',
@@ -91,6 +92,7 @@
# review-team optionally added by leaper.py.
'repo-checker': 'repo-checker',
'repo_checker-arch-whitelist': 'x86_64',
+ 'repo_checker-no-filter': 'True',
# 16 hour staging window for follow-ups since lower throughput.
'splitter-staging-age-max': '57600',
# No special packages since they will pass through SLE first.
@@ -115,6 +117,7 @@
'main-repo': 'standard',
'leaper-override-group': 'leap-reviewers',
'repo_checker-arch-whitelist': 'x86_64',
+ 'repo_checker-no-filter': 'True',
},
r'openSUSE:(?P<project>Backports:(?P<version>[^:]+))$': {
'staging': 'openSUSE:%(project)s:Staging',
@@ -153,7 +156,6 @@
'lock': None,
'lock-ns': None,
'delreq-review': None,
- 'main-repo': 'openSUSE_Factory',
'_priority': '0', # Apply defaults first
},
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20180820.d7d5724/osclib/core.py
new/openSUSE-release-tools-20180822.a9f1bc0/osclib/core.py
--- old/openSUSE-release-tools-20180820.d7d5724/osclib/core.py 2018-08-21
04:03:44.000000000 +0200
+++ new/openSUSE-release-tools-20180822.a9f1bc0/osclib/core.py 2018-08-23
00:33:11.000000000 +0200
@@ -13,6 +13,7 @@
from urllib2 import HTTPError
from osc.core import get_binarylist
+from osc.core import get_commitlog
from osc.core import get_dependson
from osc.core import http_GET
from osc.core import http_POST
@@ -22,6 +23,7 @@
from osc.core import Request
from osc.core import show_package_meta
from osc.core import show_project_meta
+from osc.core import show_results_meta
from osclib.conf import Config
from osclib.memoize import memoize
@@ -89,18 +91,14 @@
return sorted(packages)
@memoize(session=True)
-def target_archs(apiurl, project):
- meta = show_project_meta(apiurl, project)
- meta = ET.fromstringlist(meta)
- archs = []
- for arch in meta.findall('repository[@name="standard"]/arch'):
- archs.append(arch.text)
- return archs
+def target_archs(apiurl, project, repository='standard'):
+ meta = ETL.fromstringlist(show_project_meta(apiurl, project))
+ return meta.xpath('repository[@name="{}"]/arch/text()'.format(repository))
@memoize(session=True)
def depends_on(apiurl, project, repository, packages=None, reverse=None):
dependencies = set()
- for arch in target_archs(apiurl, project):
+ for arch in target_archs(apiurl, project, repository):
root = ET.fromstring(get_dependson(apiurl, project, repository, arch,
packages, reverse))
dependencies.update(pkgdep.text for pkgdep in
root.findall('.//pkgdep'))
@@ -114,18 +112,6 @@
return date_parse(when)
-def request_staged(request):
- for review in request.reviews:
- if (review.state == 'new' and review.by_project and
- review.by_project.startswith(request.actions[0].tgt_project)):
-
- # Allow time for things to settle.
- when = request_when_staged(request, review.by_project)
- if (datetime.utcnow() - when).total_seconds() > 10 * 60:
- return review.by_project
-
- return None
-
def binary_list(apiurl, project, repository, arch, package=None):
parsed = []
for binary in get_binarylist(apiurl, project, repository, arch, package):
@@ -349,3 +335,80 @@
# The OBS API of attributes is super strange, POST to update.
url = makeurl(apiurl, ['source', project, '_attribute'])
http_POST(url, data=ET.tostring(root))
+
+@memoize(session=True)
+def repository_path_expand(apiurl, project, repo, repos=None):
+ """Recursively list underlying projects."""
+
+ if repos is None:
+ # Avoids screwy behavior where list as default shares reference for all
+ # calls which effectively means the list grows even when new project.
+ repos = []
+
+ if [project, repo] in repos:
+ # For some reason devel projects such as graphics include the same path
+ # twice for openSUSE:Factory/snapshot. Does not hurt anything, but
+ # cleaner not to include it twice.
+ return repos
+
+ repos.append([project, repo])
+
+ meta = ET.fromstringlist(show_project_meta(apiurl, project))
+ for path in meta.findall('.//repository[@name="{}"]/path'.format(repo)):
+ repository_path_expand(apiurl, path.get('project', project),
path.get('repository'), repos)
+
+ return repos
+
+@memoize(session=True)
+def repository_path_search(apiurl, project, search_project, search_repository):
+ queue = []
+
+ # Initialize breadth first search queue with repositories from top project.
+ root = ETL.fromstringlist(show_project_meta(apiurl, project))
+ for repository in root.xpath('repository[path[@project and
@repository]]/@name'):
+ queue.append((repository, project, repository))
+
+ # Perform a breadth first search and return the first repository chain with
+ # a series of path elements targeting search project and repository.
+ for repository_top, project, repository in queue:
+ if root.get('name') != project:
+ # Repositories for a single project are in a row so cache parsing.
+ root = ETL.fromstringlist(show_project_meta(apiurl, project))
+
+ paths = root.findall('repository[@name="{}"]/path'.format(repository))
+ for path in paths:
+ if path.get('project') == search_project and
path.get('repository') == search_repository:
+ return repository_top
+
+ queue.append((repository_top, path.get('project'),
path.get('repository')))
+
+ return None
+
+def repository_state(apiurl, project, repository):
+ return ET.fromstringlist(show_results_meta(
+ apiurl, project, multibuild=True,
repository=[repository])).get('state')
+
+def repositories_states(apiurl, repository_pairs):
+ states = []
+
+ for project, repository in repository_pairs:
+ states.append(repository_state(apiurl, project, repository))
+
+ return states
+
+def repository_published(apiurl, project, repository):
+ root = ETL.fromstringlist(show_results_meta(
+ apiurl, project, multibuild=True, repository=[repository]))
+ return not len(root.xpath('result[@state!="published" and
@state!="unpublished"]'))
+
+def repositories_published(apiurl, repository_pairs):
+ for project, repository in repository_pairs:
+ if not repository_published(apiurl, project, repository):
+ return (project, repository)
+
+ return True
+
+def project_meta_revision(apiurl, project):
+ root = ET.fromstringlist(get_commitlog(
+ apiurl, project, '_project', None, format='xml', meta=True))
+ return int(root.find('logentry').get('revision'))
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20180820.d7d5724/osclib/cycle.py
new/openSUSE-release-tools-20180822.a9f1bc0/osclib/cycle.py
--- old/openSUSE-release-tools-20180820.d7d5724/osclib/cycle.py 2018-08-21
04:03:44.000000000 +0200
+++ new/openSUSE-release-tools-20180822.a9f1bc0/osclib/cycle.py 2018-08-23
00:33:11.000000000 +0200
@@ -202,15 +202,12 @@
graph.subpkgs = subpkgs
return graph
- def cycles(self, staging, project=None, repository='standard',
arch='x86_64'):
+ def cycles(self, override_pair, overridden_pair, arch):
"""Detect cycles in a specific repository."""
- if not project:
- project = self.api.project
-
# Detect cycles - We create the full graph from _builddepinfo.
- project_graph = self._get_builddepinfo_graph(project, repository, arch)
- current_graph = self._get_builddepinfo_graph(staging, repository, arch)
+ project_graph = self._get_builddepinfo_graph(overridden_pair[0],
overridden_pair[1], arch)
+ current_graph = self._get_builddepinfo_graph(override_pair[0],
override_pair[1], arch)
# Sometimes, new cycles have only new edges, but not new
# packages. We need to inform about this, so this can become
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20180820.d7d5724/osclib/stagingapi.py
new/openSUSE-release-tools-20180822.a9f1bc0/osclib/stagingapi.py
--- old/openSUSE-release-tools-20180820.d7d5724/osclib/stagingapi.py
2018-08-21 04:03:44.000000000 +0200
+++ new/openSUSE-release-tools-20180822.a9f1bc0/osclib/stagingapi.py
2018-08-23 00:33:11.000000000 +0200
@@ -1792,15 +1792,3 @@
return meta.find(xpath) is not None
return False
-
- # recursively detect underlying projects
- def expand_project_repo(self, project, repo, repos):
- repos.append([project, repo])
- url = self.makeurl(['source', project, '_meta'])
- meta = ET.parse(self.retried_GET(url)).getroot()
- for path in
meta.findall('.//repository[@name="{}"]/path'.format(repo)):
- self.expand_project_repo(path.get('project', project),
path.get('repository'), repos)
- return repos
-
- def expanded_repos(self, repo):
- return self.expand_project_repo(self.project, repo, [])
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20180820.d7d5724/osclib/util.py
new/openSUSE-release-tools-20180822.a9f1bc0/osclib/util.py
--- old/openSUSE-release-tools-20180820.d7d5724/osclib/util.py 2018-08-21
04:03:44.000000000 +0200
+++ new/openSUSE-release-tools-20180822.a9f1bc0/osclib/util.py 2018-08-23
00:33:11.000000000 +0200
@@ -104,3 +104,11 @@
s = smtplib.SMTP(config.get('mail-relay', 'relay.suse.de'))
s.sendmail(msg['From'], [msg['To']], msg.as_string())
s.quit()
+
+def sha1_short(data):
+ import hashlib
+
+ if isinstance(data, list):
+ data = '::'.join(data)
+
+ return hashlib.sha1(data).hexdigest()[:7]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20180820.d7d5724/pkglistgen.py
new/openSUSE-release-tools-20180822.a9f1bc0/pkglistgen.py
--- old/openSUSE-release-tools-20180820.d7d5724/pkglistgen.py 2018-08-21
04:03:44.000000000 +0200
+++ new/openSUSE-release-tools-20180822.a9f1bc0/pkglistgen.py 2018-08-23
00:33:11.000000000 +0200
@@ -40,6 +40,7 @@
from osc.core import undelete_package
from osc import conf
from osclib.conf import Config, str2bool
+from osclib.core import repository_path_expand
from osclib.stagingapi import StagingAPI
from osclib.util import project_list_family
from osclib.util import project_list_family_prior
@@ -525,7 +526,7 @@
g.ignore(self.groups[e])
def expand_repos(self, project, repo='standard'):
- return StagingAPI(self.apiurl, project).expanded_repos(repo)
+ return repository_path_expand(self.apiurl, project, repo)
def _check_supplements(self):
tocheck = set()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20180820.d7d5724/repo_checker.py
new/openSUSE-release-tools-20180822.a9f1bc0/repo_checker.py
--- old/openSUSE-release-tools-20180820.d7d5724/repo_checker.py 2018-08-21
04:03:44.000000000 +0200
+++ new/openSUSE-release-tools-20180822.a9f1bc0/repo_checker.py 2018-08-23
00:33:11.000000000 +0200
@@ -14,21 +14,26 @@
import sys
import tempfile
-from osclib.comments import CommentAPI
from osclib.conf import Config
from osclib.conf import str2bool
-from osclib.core import binary_list
from osclib.core import BINARY_REGEX
from osclib.core import depends_on
from osclib.core import devel_project_fallback
from osclib.core import fileinfo_ext_all
from osclib.core import package_binary_list
+from osclib.core import project_meta_revision
+from osclib.core import project_pseudometa_file_ensure
+from osclib.core import project_pseudometa_file_load
from osclib.core import project_pseudometa_package
-from osclib.core import request_staged
+from osclib.core import repository_path_search
+from osclib.core import repository_path_expand
+from osclib.core import repositories_states
+from osclib.core import repositories_published
from osclib.core import target_archs
from osclib.cycle import CycleDetector
from osclib.memoize import CACHEDIR
from osclib.memoize import memoize
+from osclib.util import sha1_short
import ReviewBot
@@ -37,67 +42,29 @@
INSTALL_REGEX = r"^(?:can't install (.*?)|found conflict of (.*?) with
(.*?)):$"
InstallSection = namedtuple('InstallSection', ('binaries', 'text'))
+ERROR_REPO_SPECIFIED = 'a repository must be specified via OSRT:Config
main-repo for {}'
+
class RepoChecker(ReviewBot.ReviewBot):
def __init__(self, *args, **kwargs):
ReviewBot.ReviewBot.__init__(self, *args, **kwargs)
# ReviewBot options.
- self.only_one_action = True
self.request_default_return = True
self.comment_handler = True
# RepoChecker options.
self.skip_cycle = False
self.force = False
- self.limit_group = None
-
- def repository_published(self, project):
- root = ET.fromstringlist(show_results_meta(
- self.apiurl, project, multibuild=True, repository=['standard']))
- return not len(root.xpath('result[@state!="published"]'))
def project_only(self, project, post_comments=False):
- api = self.staging_api(project)
-
- if not self.force and not self.repository_published(project):
- self.logger.info('{}/standard not published'.format(project))
+ repository = self.project_repository(project)
+ if not repository:
+ self.logger.error(ERROR_REPO_SPECIFIED.format(project))
return
- build = ET.fromstringlist(show_results_meta(
- self.apiurl, project, multibuild=True,
repository=['standard'])).get('state')
- pseudometa_content = api.pseudometa_file_load('repo_checker')
- if not self.force and pseudometa_content:
- build_previous = pseudometa_content.splitlines()[0]
- if build == build_previous:
- self.logger.info('{} build unchanged'.format(project))
- return
-
- comment = [build]
- for arch in self.target_archs(project):
- directories = []
- repos = self.staging_api(project).expanded_repos('standard')
- for layered_project, repo in repos:
- if repo != 'standard':
- raise "We assume all is standard"
- directories.append(self.mirror(layered_project, arch))
-
- parse = project if post_comments else False
- results = {
- 'cycle': CheckResult(True, None),
- 'install': self.install_check(project, directories, arch,
parse=parse, no_filter=True),
- }
-
- if not all(result.success for _, result in results.items()):
- self.result_comment(arch, results, comment)
-
- text = '\n'.join(comment).strip()
- if not self.dryrun:
- api.pseudometa_file_ensure('repo_checker', text + '\n',
'project_only run')
- else:
- print(text)
-
- if post_comments:
- self.package_comments(project)
+ repository_pairs = repository_path_expand(self.apiurl, project,
repository)
+ state_hash = self.repository_state(repository_pairs)
+ self.repository_check(repository_pairs, state_hash, False)
def package_comments(self, project):
self.logger.info('{} package
comments'.format(len(self.package_results)))
@@ -137,192 +104,26 @@
self.comment_write(state='seen', result=reference,
bot_name_suffix=bot_name_suffix,
project=comment_project,
package=comment_package, message=message)
- def prepare_review(self):
- # Reset for request batch.
- self.requests_map = {}
- self.groups = {}
- self.groups_build = {}
- self.groups_skip_cycle = []
-
- # Manipulated in ensure_group().
- self.group = None
- self.mirrored = set()
-
- # Stores parsed install_check() results grouped by package.
- self.package_results = {}
-
- # Look for requests of interest and group by staging.
- skip_build = set()
- for request in self.requests:
- # Only interesting if request is staged.
- group = request_staged(request)
- if not group:
- self.logger.debug('{}: not staged'.format(request.reqid))
- continue
-
- if self.limit_group and group != self.limit_group:
- continue
-
- # Only interested if group has completed building.
- api = self.staging_api(request.actions[0].tgt_project)
- status = api.project_status(group, True)
- # Corrupted requests may reference non-existent projects and will
- # thus return a None status which should be considered not ready.
- if not status or str(status['overall_state']) not in ('testing',
'review', 'acceptable'):
- # Not in a "ready" state.
- openQA_only = False # Not relevant so set to False.
- if status and str(status['overall_state']) == 'failed':
- # Exception to the rule is openQA only in failed state,
- # Broken packages so not just openQA.
- openQA_only = (len(status['broken_packages']) == 0)
-
- if not self.force and not openQA_only:
- self.logger.debug('{}: {} not ready'.format(request.reqid,
group))
- continue
-
- # Only interested if request is in consistent state.
- selected = api.project_status_requests('selected')
- if request.reqid not in selected:
- self.logger.debug('{}: inconsistent
state'.format(request.reqid))
-
- if group not in self.groups_build:
- # Generate build hash based on hashes from relevant projects.
- builds = []
- builds.append(ET.fromstringlist(show_results_meta(
- self.apiurl, group, multibuild=True,
repository=['standard'])).get('state'))
- builds.append(ET.fromstringlist(show_results_meta(
- self.apiurl, api.project, multibuild=True,
repository=['standard'])).get('state'))
-
- # Include meta revision for config changes (like whitelist).
- builds.append(str(api.get_prj_meta_revision(group)))
- self.groups_build[group] =
hashlib.sha1(''.join(builds)).hexdigest()[:7]
-
- # Determine if build has changed since last comment.
- comments = self.comment_api.get_comments(project_name=group)
- _, info = self.comment_api.comment_find(comments,
self.bot_name)
- if info and self.groups_build[group] == info.get('build'):
- skip_build.add(group)
-
- # Look for skip-cycle comment command.
- users =
self.request_override_check_users(request.actions[0].tgt_project)
- for _, who in self.comment_api.command_find(
- comments, self.review_user, 'skip-cycle', users):
- self.logger.debug('comment command: skip-cycle by
{}'.format(who))
- self.groups_skip_cycle.append(group)
- break
-
- if not self.force and group in skip_build:
- self.logger.debug('{}: {} build
unchanged'.format(request.reqid, group))
- continue
-
- self.requests_map[int(request.reqid)] = group
-
- requests = self.groups.get(group, [])
- requests.append(request)
- self.groups[group] = requests
-
- self.logger.debug('{}: {} ready'.format(request.reqid, group))
-
- # Filter out undesirable requests and ensure requests are ordered
- # together with group for efficiency.
- count_before = len(self.requests)
- self.requests = []
- for group, requests in sorted(self.groups.items()):
- self.requests.extend(requests)
-
- self.logger.debug('requests: {} skipped, {} queued'.format(
- count_before - len(self.requests), len(self.requests)))
-
- def ensure_group(self, request, action):
- project = action.tgt_project
- group = self.requests_map[int(request.reqid)]
-
- if group == self.group:
- # Only process a group the first time it is encountered.
- return self.group_pass
-
- self.logger.info('group {}'.format(group))
- self.group = group
- self.group_pass = True
-
- comment = []
- for arch in self.target_archs(project):
- stagings = []
- directories = []
- ignore = set()
-
- if arch not in self.target_archs(group):
- self.logger.debug('{}/{} not available'.format(group, arch))
- else:
- stagings.append(group)
- directories.append(self.mirror(group, arch))
- ignore.update(self.ignore_from_staging(project, group, arch))
-
- if not len(stagings):
- continue
-
- # Only bother if staging can match arch, but layered first.
- repos = self.staging_api(project).expanded_repos('standard')
- for layered_project, repo in repos:
- if repo != 'standard':
- raise "We assume all is standard"
- directories.append(self.mirror(layered_project, arch))
-
- whitelist = self.binary_whitelist(project, arch, group)
-
- # Perform checks on group.
- results = {
- 'cycle': self.cycle_check(project, stagings, arch),
- 'install': self.install_check(project, directories, arch,
ignore, whitelist),
- }
-
- if not all(result.success for _, result in results.items()):
- # Not all checks passed, build comment.
- self.group_pass = False
- self.result_comment(arch, results, comment)
-
- info_extra = {'build': self.groups_build[group]}
- if not self.group_pass:
- # Some checks in group did not pass, post comment.
- # Avoid identical comments with different build hash during target
- # project build phase. Once published update regardless.
- published = self.repository_published(project)
- self.comment_write(state='seen', result='failed', project=group,
- message='\n'.join(comment).strip(),
identical=True,
- info_extra=info_extra,
info_extra_identical=published)
- else:
- # Post passed comment only if previous failed comment.
- text = 'Previously reported problems have been resolved.'
- self.comment_write(state='done', result='passed', project=group,
- message=text, identical=True, only_replace=True,
- info_extra=info_extra)
-
- return self.group_pass
-
- def target_archs(self, project):
- archs = target_archs(self.apiurl, project)
+ def target_archs(self, project, repository):
+ archs = target_archs(self.apiurl, project, repository)
# Check for arch whitelist and use intersection.
- product = project.split(':Staging:', 1)[0]
- whitelist = Config.get(self.apiurl,
product).get('repo_checker-arch-whitelist')
+ whitelist = Config.get(self.apiurl,
project).get('repo_checker-arch-whitelist')
if whitelist:
archs = list(set(whitelist.split(' ')).intersection(set(archs)))
# Trick to prioritize x86_64.
return sorted(archs, reverse=True)
- def mirror(self, project, arch):
+ @memoize(ttl=60, session=True, add_invalidate=True)
+ def mirror(self, project, repository, arch):
"""Call bs_mirrorfull script to mirror packages."""
- directory = os.path.join(CACHEDIR, project, 'standard', arch)
- if (project, arch) in self.mirrored:
- # Only mirror once per request batch.
- return directory
-
+ directory = os.path.join(CACHEDIR, project, repository, arch)
if not os.path.exists(directory):
os.makedirs(directory)
script = os.path.join(SCRIPT_PATH, 'bs_mirrorfull')
- path = '/'.join((project, 'standard', arch))
+ path = '/'.join((project, repository, arch))
url = '{}/public/build/{}'.format(self.apiurl, path)
parts = ['LC_ALL=C', 'perl', script, '--nodebug', url, directory]
parts = [pipes.quote(part) for part in parts]
@@ -331,26 +132,25 @@
if os.system(' '.join(parts)):
raise Exception('failed to mirror {}'.format(path))
- self.mirrored.add((project, arch))
return directory
- def ignore_from_staging(self, project, staging, arch):
- """Determine the target project binaries to ingore in favor of
staging."""
- _, binary_map = package_binary_list(self.apiurl, staging, 'standard',
arch)
+ def simulated_merge_ignore(self, override_pair, overridden_pair, arch):
+ """Determine the list of binaries to similate overides in overridden
layer."""
+ _, binary_map = package_binary_list(self.apiurl, override_pair[0],
override_pair[1], arch)
packages = set(binary_map.values())
- binaries, _ = package_binary_list(self.apiurl, project, 'standard',
arch)
+ binaries, _ = package_binary_list(self.apiurl, overridden_pair[0],
overridden_pair[1], arch)
for binary in binaries:
if binary.package in packages:
yield binary.name
@memoize(session=True)
- def binary_list_existing_problem(self, project):
+ def binary_list_existing_problem(self, project, repository):
"""Determine which binaries are mentioned in repo_checker output."""
binaries = set()
- api = self.staging_api(project)
- content = api.pseudometa_file_load('repo_checker')
+ filename = self.project_pseudometa_file_name(project, repository)
+ content = project_pseudometa_file_load(self.apiurl, project, filename)
if not content:
self.logger.warn('no project_only run from which to extract
existing problems')
return binaries
@@ -364,23 +164,29 @@
return binaries
- def binary_whitelist(self, project, arch, group):
- additions =
self.staging_api(project).get_prj_pseudometa(group).get('config', {})
- whitelist = self.binary_list_existing_problem(project)
- prefix = 'repo_checker-binary-whitelist'
- for key in [prefix, '-'.join([prefix, arch])]:
- whitelist.update(additions.get(key, '').split(' '))
+ def binary_whitelist(self, override_pair, overridden_pair, arch):
+ whitelist = self.binary_list_existing_problem(overridden_pair[0],
overridden_pair[1])
+
+ if Config.get(self.apiurl, overridden_pair[0]).get('staging'):
+ additions =
self.staging_api(overridden_pair[0]).get_prj_pseudometa(
+ override_pair[0]).get('config', {})
+ prefix = 'repo_checker-binary-whitelist'
+ for key in [prefix, '-'.join([prefix, arch])]:
+ whitelist.update(additions.get(key, '').split(' '))
+
whitelist = filter(None, whitelist)
return whitelist
- def install_check(self, project, directories, arch, ignore=[],
whitelist=[], parse=False, no_filter=False):
- self.logger.info('install check: start')
+ def install_check(self, project, directories, repository, arch,
ignore=None, whitelist=[], parse=False, no_filter=False):
+ self.logger.info('install check: start (ignore:{}, whitelist:{},
parse:{}, no_filter:{})'.format(
+ bool(ignore), len(whitelist), bool(parse), no_filter))
with tempfile.NamedTemporaryFile() as ignore_file:
# Print ignored rpms on separate lines in ignore file.
- for item in ignore:
- ignore_file.write(item + '\n')
- ignore_file.flush()
+ if ignore:
+ for item in ignore:
+ ignore_file.write(item + '\n')
+ ignore_file.flush()
# Invoke repo_checker.pl to perform an install check.
script = os.path.join(SCRIPT_PATH, 'repo_checker.pl')
@@ -399,11 +205,11 @@
self.logger.info('install check: failed')
if p.returncode == 126:
self.logger.warn('mirror cache reset due to corruption')
- self.mirrored = set()
+ self._invalidate_all()
elif parse:
# Parse output for later consumption for posting comments.
sections = self.install_check_parse(stdout)
- self.install_check_sections_group(parse, arch, sections)
+ self.install_check_sections_group(parse, repository, arch,
sections)
# Format output as markdown comment.
parts = []
@@ -416,16 +222,16 @@
parts.append('<pre>\n' + stderr + '\n' + '</pre>\n')
pseudometa_project, pseudometa_package =
project_pseudometa_package(self.apiurl, project)
- path = ['package', 'view_file', pseudometa_project,
pseudometa_package, 'repo_checker']
+ filename = self.project_pseudometa_file_name(project, repository)
+ path = ['package', 'view_file', pseudometa_project,
pseudometa_package, filename]
header = '### [install check & file
conflicts](/{})\n\n'.format('/'.join(path))
return CheckResult(False, header + ('\n' + ('-' * 80) +
'\n\n').join(parts))
-
self.logger.info('install check: passed')
return CheckResult(True, None)
- def install_check_sections_group(self, project, arch, sections):
- _, binary_map = package_binary_list(self.apiurl, project, 'standard',
arch)
+ def install_check_sections_group(self, project, repository, arch,
sections):
+ _, binary_map = package_binary_list(self.apiurl, project, repository,
arch)
for section in sections:
# If switch to creating bugs likely makes sense to join packages to
@@ -463,36 +269,52 @@
if section:
yield InstallSection(section, text)
- def cycle_check(self, project, stagings, arch):
- if self.skip_cycle or self.group in self.groups_skip_cycle:
+ @memoize(ttl=60, session=True)
+ def cycle_check_skip(self, project):
+ if self.skip_cycle:
+ return True
+
+ # Look for skip-cycle comment command.
+ comments = self.comment_api.get_comments(project_name=project)
+ users = self.request_override_check_users(project)
+ for _, who in self.comment_api.command_find(
+ comments, self.review_user, 'skip-cycle', users):
+ self.logger.debug('comment command: skip-cycle by {}'.format(who))
+ return True
+
+ return False
+
+ def cycle_check(self, override_pair, overridden_pair, arch):
+ if self.cycle_check_skip(override_pair[0]):
self.logger.info('cycle check: skip due to --skip-cycle or comment
command')
return CheckResult(True, None)
self.logger.info('cycle check: start')
- cycle_detector = CycleDetector(self.staging_api(project))
comment = []
- for staging in stagings:
- first = True
- for index, (cycle, new_edges, new_packages) in enumerate(
- cycle_detector.cycles(staging, arch=arch), start=1):
- if not new_packages:
- continue
+ first = True
+ cycle_detector = CycleDetector(self.staging_api(overridden_pair[0]))
+ for index, (cycle, new_edges, new_packages) in enumerate(
+ cycle_detector.cycles(override_pair, overridden_pair, arch),
start=1):
+
+ if not new_packages:
+ continue
- if first:
- comment.append('### new
[cycle(s)](/project/repository_state/{}/standard)\n'.format(staging))
- first = False
-
- # New package involved in cycle, build comment.
- comment.append('- #{}: {} package cycle, {} new edges'.format(
- index, len(cycle), len(new_edges)))
-
- comment.append(' - cycle')
- for package in sorted(cycle):
- comment.append(' - {}'.format(package))
-
- comment.append(' - new edges')
- for edge in sorted(new_edges):
- comment.append(' - ({}, {})'.format(edge[0], edge[1]))
+ if first:
+ comment.append('### new
[cycle(s)](/project/repository_state/{}/{})\n'.format(
+ override_pair[0], override_pair[1]))
+ first = False
+
+ # New package involved in cycle, build comment.
+ comment.append('- #{}: {} package cycle, {} new edges'.format(
+ index, len(cycle), len(new_edges)))
+
+ comment.append(' - cycle')
+ for package in sorted(cycle):
+ comment.append(' - {}'.format(package))
+
+ comment.append(' - new edges')
+ for edge in sorted(new_edges):
+ comment.append(' - ({}, {})'.format(edge[0], edge[1]))
if len(comment):
# New cycles, post comment.
@@ -502,15 +324,210 @@
self.logger.info('cycle check: passed')
return CheckResult(True, None)
- def result_comment(self, arch, results, comment):
+ def result_comment(self, repository, arch, results, comment):
"""Generate comment from results"""
- comment.append('## ' + arch + '\n')
+ comment.append('## {}/{}\n'.format(repository, arch))
for result in results.values():
if not result.success:
comment.append(result.comment)
+ def project_pseudometa_file_name(self, project, repository):
+ filename = 'repo_checker'
+
+ main_repo = Config.get(self.apiurl, project).get('main-repo')
+ if not main_repo:
+ filename += '.' + repository
+
+ return filename
+
+ @memoize(ttl=60, session=True)
+ def repository_state(self, repository_pairs):
+ states = repositories_states(self.apiurl, repository_pairs)
+ states.append(str(project_meta_revision(self.apiurl,
repository_pairs[0][0])))
+
+ return sha1_short(states)
+
+ @memoize(ttl=60, session=True)
+ def repository_state_last(self, project, repository, pseudometa):
+ if pseudometa:
+ filename = self.project_pseudometa_file_name(project, repository)
+ content = project_pseudometa_file_load(self.apiurl, project,
filename)
+ if content:
+ return content.splitlines()[0]
+ else:
+ comments = self.comment_api.get_comments(project_name=project)
+ _, info = self.comment_api.comment_find(comments, self.bot_name)
+ if info:
+ return info.get('build')
+
+ return None
+
+ @memoize(session=True)
+ def repository_check(self, repository_pairs, state_hash, simulate_merge,
post_comments=False):
+ comment = []
+ project, repository = repository_pairs[0] # this would mean staging!?
+ self.logger.info('checking {}/{}@{}[{}]'.format(
+ project, repository, state_hash, len(repository_pairs)))
+
+ published = repositories_published(self.apiurl, repository_pairs)
+
+ if not self.force:
+ if state_hash == self.repository_state_last(project, repository,
not simulate_merge):
+ self.logger.info('{} build unchanged'.format(project))
+ # TODO keep track of skipped count for cycle summary
+ return None
+
+ # For submit style requests, want to process if top layer is done,
+ # but not mark review as final until all layers are published.
+ if published is not True and (not simulate_merge or published[0]
== project):
+ # Require all layers to be published except when the top layer
+ # is published in a simulate merge (allows quicker feedback
with
+ # potentially incorrect resutls for staging).
+ self.logger.info('{}/{} not published'.format(published[0],
published[1]))
+ return None
+
+ # Drop non-published repository information and thus reduce to boolean.
+ published = published is True
+
+ if simulate_merge:
+ # Restrict top layer archs to the whitelisted archs from merge
layer.
+ archs = set(target_archs(self.apiurl, project,
repository)).intersection(
+ set(self.target_archs(repository_pairs[1][0],
repository_pairs[1][1])))
+ else:
+ # Top of pseudometa file.
+ comment.append(state_hash)
+ archs = self.target_archs(project, repository)
+
+ if post_comments:
+ # Stores parsed install_check() results grouped by package.
+ self.package_results = {}
+
+ if not len(archs):
+ self.logger.debug('{} has no relevant
architectures'.format(project))
+ return None
+
+ result = True
+ for arch in archs:
+ directories = []
+ for pair_project, pair_repository in repository_pairs:
+ directories.append(self.mirror(pair_project, pair_repository,
arch))
+
+ if simulate_merge:
+ ignore = self.simulated_merge_ignore(repository_pairs[0],
repository_pairs[1], arch)
+ whitelist = self.binary_whitelist(repository_pairs[0],
repository_pairs[1], arch)
+
+ results = {
+ 'cycle': self.cycle_check(repository_pairs[0],
repository_pairs[1], arch),
+ 'install': self.install_check(project, directories,
repository_pairs[1][1], arch, ignore, whitelist),
+ }
+ else:
+ parse = project if post_comments else False
+ # Only products themselves will want no-filter or perhaps
+ # projects working on cleaning up a product.
+ no_filter = str2bool(Config.get(self.apiurl,
project).get('repo_checker-no-filter'))
+ results = {
+ 'cycle': CheckResult(True, None),
+ 'install': self.install_check(project, directories,
repository, arch,
+ parse=parse,
no_filter=no_filter),
+ }
+
+ if not all(result.success for _, result in results.items()):
+ # Not all checks passed, build comment.
+ result = False
+ self.result_comment(repository, arch, results, comment)
+
+ if simulate_merge:
+ info_extra = {'build': state_hash}
+ if not result:
+ # Some checks in group did not pass, post comment.
+ # Avoid identical comments with different build hash during
+ # target project build phase. Once published update regardless.
+ self.comment_write(state='seen', result='failed',
project=project,
+ message='\n'.join(comment).strip(),
identical=True,
+ info_extra=info_extra,
info_extra_identical=published)
+ else:
+ # Post passed comment only if previous failed comment.
+ text = 'Previously reported problems have been resolved.'
+ self.comment_write(state='done', result='passed',
project=project,
+ message=text, identical=True,
only_replace=True,
+ info_extra=info_extra)
+ else:
+ text = '\n'.join(comment).strip()
+ if not self.dryrun:
+ filename = self.project_pseudometa_file_name(project,
repository)
+ project_pseudometa_file_ensure(
+ self.apiurl, project, filename, text + '\n', 'repo_checker
project_only run')
+ else:
+ print(text)
+
+ if post_comments:
+ self.package_comments(project)
+
+ if result and not published:
+ # Wait for the complete stack to build before positive result.
+ self.logger.debug('demoting result from accept to ignore due to
non-published layer')
+ result = None
+
+ return result
+
+ @memoize(session=True)
+ def project_repository(self, project):
+ repository = Config.get(self.apiurl, project).get('main-repo')
+ if not repository:
+ self.logger.debug('no main-repo defined for {}'.format(project))
+
+ search_project = 'openSUSE:Factory'
+ for search_repository in ('snapshot', 'standard'):
+ repository = repository_path_search(
+ self.apiurl, project, search_project, search_repository)
+
+ if repository:
+ self.logger.debug('found chain to {}/{} via {}'.format(
+ search_project, search_repository, repository))
+ break
+
+ return repository
+
+ @memoize(ttl=60, session=True)
+ def request_repository_pairs(self, request, action):
+ repository = self.project_repository(action.tgt_project)
+ if not repository:
+ self.review_messages['declined'] =
ERROR_REPO_SPECIFIED.format(action.tgt_project)
+ return False
+
+ repository_pairs = []
+ # Assumes maintenance_release target project has staging disabled.
+ if Config.get(self.apiurl, action.tgt_project).get('staging'):
+ stage_info =
self.staging_api(action.tgt_project).packages_staged.get(action.tgt_package)
+ if not stage_info or str(stage_info['rq_id']) !=
str(request.reqid):
+ self.logger.info('{} not staged'.format(request.reqid))
+ return None
+
+ # Staging setup is convoluted and thus the repository setup does
not
+ # contain a path to the target project. Instead the ports
repository
+ # is used to import the target prjconf. As such the staging group
+ # repository must be explicitly layered on top of target project.
+ repository_pairs.append([stage_info['prj'], repository])
+ repository_pairs.extend(repository_path_expand(self.apiurl,
action.tgt_project, repository))
+ else:
+ # Find a repository which links to target project "main"
repository.
+ repository = repository_path_search(
+ self.apiurl, action.src_project, action.tgt_project,
repository)
+ if not repository:
+ self.review_messages['declined'] =
ERROR_REPO_SPECIFIED.format(action.tgt_project)
+ return False
+
+ repository_pairs.extend(repository_path_expand(self.apiurl,
action.src_project, repository))
+
+ return repository_pairs
+
def check_action_submit(self, request, action):
- if not self.ensure_group(request, action):
+ repository_pairs = self.request_repository_pairs(request, action)
+ if not isinstance(repository_pairs, list):
+ return repository_pairs
+
+ state_hash = self.repository_state(repository_pairs)
+ if not self.repository_check(repository_pairs, state_hash, True):
return None
self.review_messages['accepted'] = 'cycle and install check passed'
@@ -547,11 +564,30 @@
self.comment_write(state='seen', result='failed')
return None
- # Allow for delete to be declined before ensuring group passed.
- if not self.ensure_group(request, action):
+ repository_pairs = self.request_repository_pairs(request, action)
+ if not isinstance(repository_pairs, list):
+ return repository_pairs
+
+ state_hash = self.repository_state(repository_pairs)
+ if not self.repository_check(repository_pairs, state_hash, True):
return None
- self.review_messages['accepted'] = 'delete request is safe'
+ self.review_messages['accepted'] = 'cycle and install check passed'
+ return True
+
+ def check_action_maintenance_release(self, request, action):
+ # No reason to special case patchinfo since same source and target
+ # projects which is all that repo_checker cares about.
+
+ repository_pairs = self.request_repository_pairs(request, action)
+ if not isinstance(repository_pairs, list):
+ return repository_pairs
+
+ state_hash = self.repository_state(repository_pairs)
+ if not self.repository_check(repository_pairs, state_hash, True):
+ return None
+
+ self.review_messages['accepted'] = 'cycle and install check passed'
return True
@@ -566,7 +602,6 @@
parser.add_option('--skip-cycle', action='store_true', help='skip
cycle check')
parser.add_option('--force', action='store_true', help='force review
even if project is not ready')
- parser.add_option('--limit-group', metavar='GROUP', help='only review
requests in specific group')
return parser
@@ -577,7 +612,6 @@
bot.skip_cycle = self.options.skip_cycle
bot.force = self.options.force
- bot.limit_group = self.options.limit_group
return bot
++++++ openSUSE-release-tools.obsinfo ++++++
--- /var/tmp/diff_new_pack.nozftN/_old 2018-08-31 10:43:23.687127537 +0200
+++ /var/tmp/diff_new_pack.nozftN/_new 2018-08-31 10:43:23.687127537 +0200
@@ -1,5 +1,5 @@
name: openSUSE-release-tools
-version: 20180820.d7d5724
-mtime: 1534817024
-commit: d7d5724daefff290bbef79f37481a40ff6a1d57d
+version: 20180822.a9f1bc0
+mtime: 1534977191
+commit: a9f1bc0ee71e8d1307b4e756f220e0289a8bd17a
