Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package openSUSE-release-tools for
openSUSE:Factory checked in at 2022-03-23 20:19:17
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old)
and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.25692 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openSUSE-release-tools"
Wed Mar 23 20:19:17 2022 rev:408 rq:964323 version:20220323.bd336586
Changes:
--------
---
/work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes
2022-03-22 19:40:05.051082607 +0100
+++
/work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.25692/openSUSE-release-tools.changes
2022-03-23 20:21:09.626557197 +0100
@@ -1,0 +2,27 @@
+Wed Mar 23 15:42:45 UTC 2022 - opensuse-releaset...@opensuse.org
+
+- Update to version 20220323.bd336586:
+ * Remove rpmlint check from check_source
+ * Small fixes for staging-installcheck
+ * Add osc-pcheck: support devel prj maintainers
+
+-------------------------------------------------------------------
+Wed Mar 23 14:16:39 UTC 2022 - Stephan Kulow <co...@suse.com>
+
+- just testing
+
+-------------------------------------------------------------------
+Wed Mar 23 10:35:13 UTC 2022 - opensuse-releaset...@opensuse.org
+
+- Update to version 20220323.ad923e6f:
+ * Add some documentation for pkglistgen staging workflow
+ * accept: Create pkglistgen .changes entry and commit
+ * Prepare applying pkglistgen diffs on staging accept
+
+-------------------------------------------------------------------
+Tue Mar 22 14:29:18 UTC 2022 - opensuse-releaset...@opensuse.org
+
+- Update to version 20220322.e78b5352:
+ * staging-installcheck: Exit 1 for the letter stagings
+
+-------------------------------------------------------------------
Old:
----
openSUSE-release-tools-20220321.48333852.obscpio
New:
----
openSUSE-release-tools-20220323.bd336586.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ openSUSE-release-tools.spec ++++++
--- /var/tmp/diff_new_pack.PVnDVa/_old 2022-03-23 20:21:10.358557612 +0100
+++ /var/tmp/diff_new_pack.PVnDVa/_new 2022-03-23 20:21:10.362557614 +0100
@@ -20,7 +20,7 @@
%define source_dir openSUSE-release-tools
%define announcer_filename factory-package-news
Name: openSUSE-release-tools
-Version: 20220321.48333852
+Version: 20220323.bd336586
Release: 0
Summary: Tools to aid in staging and release work for openSUSE/SUSE
License: GPL-2.0-or-later AND MIT
@@ -242,6 +242,18 @@
%description -n osc-plugin-cycle
OSC plugin for cycle visualization, see `osc cycle --help`.
+%package -n osc-plugin-pcheck
+Summary: OSC plugin to support devel project maintainers
+Group: Development/Tools/Other
+Requires: osc >= 0.165.1
+Requires: osclib = %{version}
+BuildArch: noarch
+
+%description -n osc-plugin-pcheck
+OSC plugin for devel project maintainers. Helps them check the submit
+state (done, todo, missing links) of a devel project to the parent project.
+See 'osc pcheck --help'
+
%package -n osc-plugin-origin
Summary: OSC plugin for origin management
Group: Development/Tools/Other
@@ -378,6 +390,7 @@
%exclude %{_datadir}/%{source_dir}/osclib
%exclude %{_datadir}/%{source_dir}/osc-cycle.py
%exclude %{_datadir}/%{source_dir}/osc-origin.py
+%exclude %{_datadir}/%{source_dir}/osc-pcheck.py
%exclude %{_datadir}/%{source_dir}/osc-staging.py
%exclude %{_datadir}/%{source_dir}/findfileconflicts
%exclude %{_datadir}/%{source_dir}/write_repo_susetags_file.pl
@@ -470,6 +483,10 @@
%{_datadir}/%{source_dir}/osc-cycle.py
%{osc_plugin_dir}/osc-cycle.py
+%files -n osc-plugin-pcheck
+%{_datadir}/%{source_dir}/osc-pcheck.py
+%{osc_plugin_dir}/osc-pcheck.py
+
%files -n osc-plugin-origin
%{_datadir}/%{source_dir}/osc-origin.py
%{osc_plugin_dir}/osc-origin.py
++++++ _servicedata ++++++
--- /var/tmp/diff_new_pack.PVnDVa/_old 2022-03-23 20:21:10.406557639 +0100
+++ /var/tmp/diff_new_pack.PVnDVa/_new 2022-03-23 20:21:10.410557641 +0100
@@ -1,7 +1,7 @@
<servicedata>
<service name="tar_scm">
<param
name="url">https://github.com/openSUSE/openSUSE-release-tools.git</param>
- <param
name="changesrevision">1a26c0202014ca37682cffdf9125050e00f8b927</param>
+ <param
name="changesrevision">bd3365863f090bfe86f03dd73b483431c9ed33f3</param>
</service>
</servicedata>
++++++ openSUSE-release-tools-20220321.48333852.obscpio ->
openSUSE-release-tools-20220323.bd336586.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openSUSE-release-tools-20220321.48333852/CONTENTS.md
new/openSUSE-release-tools-20220323.bd336586/CONTENTS.md
--- old/openSUSE-release-tools-20220321.48333852/CONTENTS.md 2022-03-21
17:38:05.000000000 +0100
+++ new/openSUSE-release-tools-20220323.bd336586/CONTENTS.md 2022-03-23
16:41:56.000000000 +0100
@@ -286,6 +286,12 @@
* Package: --
* Usage: used to debug problems. See
https://github.com/openSUSE/openSUSE-release-tools/pull/992 as an example.
+#### osc-pcheck.py
+* Sources: [osc-pcheck.py](osc-pcheck.py)
+* Documentation: --
+* Package: --
+* Usage: Overview for devel project maintainers: unsubmitted packages with
diff, submitted packages, and unlinked packages (things to monitor)
+
#### compare_pkglist.py
Compares packages status between two projects. It determines which project has
the newer version of a package,
@@ -338,4 +344,4 @@
* Sources: [check_bugowner.py](check_bugowner.py)
* Documentation: --
* Package: openSUSE-release-tools
- * Usage: gocd
\ No newline at end of file
+ * Usage: gocd
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20220321.48333852/check_source.py
new/openSUSE-release-tools-20220323.bd336586/check_source.py
--- old/openSUSE-release-tools-20220321.48333852/check_source.py
2022-03-21 17:38:05.000000000 +0100
+++ new/openSUSE-release-tools-20220323.bd336586/check_source.py
2022-03-23 16:41:56.000000000 +0100
@@ -16,8 +16,6 @@
from osclib.core import devel_project_fallback
from osclib.core import group_members
from osclib.core import package_kind
-from osclib.core import source_file_load
-from osclib.core import target_archs
from osclib.core import create_add_role_request
from osc.core import show_project_meta
from osc.core import get_request_list
@@ -30,11 +28,6 @@
class CheckSource(ReviewBot.ReviewBot):
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
- AUDIT_BUG_URL =
"https://en.opensuse.org/openSUSE:Package_security_guidelines#audit_bugs";
- AUDIT_BUG_MESSAGE = """The package is submitted to an official product and
it has warnings that indicate
-that it need to go through a security review.
-Those warnings can only be ignored in devel projects. For more information
please read: {}.""".format(
- AUDIT_BUG_URL)
def __init__(self, *args, **kwargs):
ReviewBot.ReviewBot.__init__(self, *args, **kwargs)
@@ -59,8 +52,6 @@
self.required_maintainer = config.get('required-source-maintainer', '')
self.devel_whitelist = config.get('devel-whitelist', '').split()
self.skip_add_reviews = False
- self.security_review_team = config.get('security-review-team',
'security-team')
- self.bad_rpmlint_entries = config.get('bad-rpmlint-entries',
'').split()
if self.action.type == 'maintenance_incident':
# The workflow effectively enforces the names to match and the
@@ -265,68 +256,8 @@
elif self.repo_checker is not None:
self.add_review(self.request, by_user=self.repo_checker,
msg='Please review build success')
- if self.bad_rpmlint_entries:
- warnings = self.has_whitelist_warnings(source_project,
source_package, target_project, target_package)
- if warnings:
- # if there are any add a review for the security team
- # maybe add the found warnings to the message for the review
- message = CheckSource.AUDIT_BUG_MESSAGE + \
- "\nTriggered by whitelist
warnings:\n{}".format("\n".join(warnings))
- self.add_review(self.request,
by_group=self.security_review_team, msg=message)
- warnings = self.suppresses_whitelist_warnings(source_project,
source_package)
- if warnings:
- message = CheckSource.AUDIT_BUG_MESSAGE + \
- "\nTriggered by suppressed whitelist
warning:\n{}".format("\n".join(warnings))
- self.add_review(self.request,
by_group=self.security_review_team, msg=message)
-
return True
- def suppresses_whitelist_warnings(self, source_project, source_package):
- # checks if there's a rpmlintrc that suppresses warnings that we check
- found_entries = set()
- contents = source_file_load(self.apiurl, source_project,
source_package, source_package + '-rpmlintrc')
- if contents:
- contents = re.sub(r'(?m)^ *#.*\n?', '', contents)
- matches = re.findall(r'addFilter\(["\']([^"\']+)["\']\)', contents)
- # this is a bit tricky. Since users can specify arbitrary regular
expresions it's not easy
- # to match bad_rpmlint_entries against what we found
- for entry in self.bad_rpmlint_entries:
- for match in matches:
- # First we try to see if our entries appear verbatim in
the rpmlint entries
- if entry in match:
- self.logger.info(f'found suppressed whitelist warning:
{match}')
- found_entries.add(match)
- # if that's not the case then we check if one of the
entries in the rpmlint file would match one
- # of our entries (e.g. addFilter(".*")
- elif re.search(match, entry) and match not in
found_entries:
- self.logger.info(f'found rpmlint entry that suppresses
an important warning: {match}')
- found_entries.add(match)
-
- return found_entries
-
- def has_whitelist_warnings(self, source_project, source_package,
target_project, target_package):
- # this checks if this is a submit to an product project and it has
warnings for non-whitelisted permissions/files
- found_entries = set()
- url = osc.core.makeurl(self.apiurl, ['build', target_project])
- xml = ET.parse(osc.core.http_GET(url)).getroot()
- for f in xml.findall('entry'):
- # we check all repos in the source project for errors that exist
in the target project
- repo = f.attrib['name']
- query = {'last': 1, }
- for arch in target_archs(self.apiurl, source_project, repo):
- url = osc.core.makeurl(self.apiurl, ['build', source_project,
repo,
- arch, source_package, '_log'],
query=query)
- try:
- result = osc.core.http_GET(url)
- contents = str(result.read())
- for entry in self.bad_rpmlint_entries:
- if (': W: ' + entry in contents) and not (entry in
found_entries):
- self.logger.info(f'found missing whitelist for
warning: {entry}')
- found_entries.add(entry)
- except HTTPError as e:
- self.logger.info('ERROR in URL %s [%s]' % (url, e))
- return found_entries
-
def is_devel_project(self, source_project, target_project):
if source_project in self.devel_whitelist:
return True
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20220321.48333852/dist/package/openSUSE-release-tools.spec
new/openSUSE-release-tools-20220323.bd336586/dist/package/openSUSE-release-tools.spec
---
old/openSUSE-release-tools-20220321.48333852/dist/package/openSUSE-release-tools.spec
2022-03-21 17:38:05.000000000 +0100
+++
new/openSUSE-release-tools-20220323.bd336586/dist/package/openSUSE-release-tools.spec
2022-03-23 16:41:56.000000000 +0100
@@ -242,6 +242,18 @@
%description -n osc-plugin-cycle
OSC plugin for cycle visualization, see `osc cycle --help`.
+%package -n osc-plugin-pcheck
+Summary: OSC plugin to support devel project maintainers
+Group: Development/Tools/Other
+Requires: osc >= 0.165.1
+Requires: osclib = %{version}
+BuildArch: noarch
+
+%description -n osc-plugin-pcheck
+OSC plugin for devel project maintainers. Helps them check the submit
+state (done, todo, missing links) of a devel project to the parent project.
+See 'osc pcheck --help'
+
%package -n osc-plugin-origin
Summary: OSC plugin for origin management
Group: Development/Tools/Other
@@ -378,6 +390,7 @@
%exclude %{_datadir}/%{source_dir}/osclib
%exclude %{_datadir}/%{source_dir}/osc-cycle.py
%exclude %{_datadir}/%{source_dir}/osc-origin.py
+%exclude %{_datadir}/%{source_dir}/osc-pcheck.py
%exclude %{_datadir}/%{source_dir}/osc-staging.py
%exclude %{_datadir}/%{source_dir}/findfileconflicts
%exclude %{_datadir}/%{source_dir}/write_repo_susetags_file.pl
@@ -470,6 +483,10 @@
%{_datadir}/%{source_dir}/osc-cycle.py
%{osc_plugin_dir}/osc-cycle.py
+%files -n osc-plugin-pcheck
+%{_datadir}/%{source_dir}/osc-pcheck.py
+%{osc_plugin_dir}/osc-pcheck.py
+
%files -n osc-plugin-origin
%{_datadir}/%{source_dir}/osc-origin.py
%{osc_plugin_dir}/osc-origin.py
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20220321.48333852/docs/pkglistgen.md
new/openSUSE-release-tools-20220323.bd336586/docs/pkglistgen.md
--- old/openSUSE-release-tools-20220321.48333852/docs/pkglistgen.md
2022-03-21 17:38:05.000000000 +0100
+++ new/openSUSE-release-tools-20220323.bd336586/docs/pkglistgen.md
2022-03-23 16:41:56.000000000 +0100
@@ -12,7 +12,7 @@
The package list generator reads several files. The most important are
group*.yml (tradionally only groups.yml) within 000package-groups.
### supportstatus.txt
- TODO
+The file lists the packages and their support level. It's only necessary to
list packages here that have a different level than the default level
specificied in the groups. The format is plain text: <package name> <level> -
the level is handed over 1:1 to KIWI file. Currently used values are:
unsupported, l2 and l3
### group*.yml
The file is a list of package lists and the special hash 'OUTPUT'. OUTPUT
contains an entry for every group file that needs to be written out. The group
name of it needs to exist as package list as well. OUTPUT also contains flags
for the groups.
@@ -90,3 +90,10 @@
## Overlap calculcation
TODO
+## Handling in staging workflow
+If 000package-groups contains a file named summary-staging.txt, the bot will
trigger a diff mode on staging projects.
+It will create an equal summary-staging.txt in 000product and create a comment
with a human readable diff in the staging
+project. This comment can be replied to. If the reply starts with 'approve',
staging accept will apply the diff to this
+txt file. If the reply starts with 'ignore', the bot will continue with the
pipeline and do nothing on staging accept.
+
+This way simple changes to the summary can be accepted without a submit
request (of which there can only be one at a time).
\ No newline at end of file
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20220321.48333852/osc-pcheck.py
new/openSUSE-release-tools-20220323.bd336586/osc-pcheck.py
--- old/openSUSE-release-tools-20220321.48333852/osc-pcheck.py 1970-01-01
01:00:00.000000000 +0100
+++ new/openSUSE-release-tools-20220323.bd336586/osc-pcheck.py 2022-03-23
16:41:56.000000000 +0100
@@ -0,0 +1,120 @@
+# Copyright (C) 2015 SUSE Linux Products GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from osc import cmdln
+import osc.core
+
+
+@cmdln.option('--push', action='store_true',
+ help="Push changed packages to their parents")
+@cmdln.option('-m', "--message",
+ help='Specify submit message')
+def do_pcheck(self, subcmd, opts, project):
+ """${cmd_name}: Show changed packages (packages that have a diff)
+
+ Examples:
+ osc pcheck <prj> # shows changed packages etc. for <prj>
+
+ --push Create submit requests for packages with a diff (if none
exists yet)
+ -m Specify submit message (defaut: "Scripted push of project
<prj>")
+
+ """
+ apiurl = self.get_api_url()
+ sinfos = osc.core.get_project_sourceinfo(apiurl, project, True)
+ todo = {}
+ errors = {}
+ md5s = {}
+ pmap = {}
+ changed = []
+ changeSRed = {}
+ api = oscapi(apiurl)
+ for pkg, sinfo in sinfos.items():
+ if sinfo.find('error'):
+ errors[pkg] = sinfo.find('error').text
+ continue
+ elif sinfo.find('originpackage') is not None:
+ # This is a package created from a _multibuild
+ # Status will be checked by the main one (which
+ # has no originpackage.) so let's not continue further
+ continue
+ elif sinfo.find('linked') is not None:
+ elm = sinfo.find('linked')
+ key = '%s/%s' % (elm.get('project'), elm.get('package'))
+ pmap.setdefault(key, []).append(pkg)
+ todo.setdefault(elm.get('project'), []).append(elm.get('package'))
+ md5s[pkg] = sinfo.get('verifymd5')
+ for prj, pkgs in todo.items():
+ sinfos = osc.core.get_project_sourceinfo(apiurl, prj, True, *pkgs)
+ for pkg, sinfo in sinfos.items():
+ key = '%s/%s' % (prj, pkg)
+ for p in pmap[key]:
+ vmd5 = md5s.pop(p)
+ if vmd5 == sinfo.get('verifymd5'):
+ continue
+ # Is there already an SR outgoing for this package?
+ SRid = int(api.sr_for_package(project, p))
+ if SRid > 0:
+ changeSRed[p] = SRid
+ else:
+ changed.append(p)
+ if opts.push:
+ if opts.message:
+ message = opts.message
+ else:
+ message = "Scripted push from
{project}".format(project=project)
+ api.create(project=project, package=p, target=prj,
message=message)
+
+ overview = 'Overview of project {}'.format(project)
+ print()
+ print(overview)
+ print('=' * len(overview))
+ print('Changed & unsubmitted packages: %d' % len(changed))
+ print(', '.join(changed))
+ print()
+ print('Changed & submitted packages: %d' % len(changeSRed.keys()))
+ print(', '.join(['%s(%s)' % (pkg, SR) for pkg, SR in changeSRed.items()]))
+ print()
+ print('Packages without link: %d' % len(md5s.keys()))
+ print(', '.join(md5s.keys()))
+ print()
+ print('Packages with errors: %d' % len(errors.keys()))
+ print('\n'.join(['%s: %s' % (p, err) for p, err in errors.items()]))
+
+
+class oscapi:
+ def __init__(self, apiurl):
+ self.apiurl = apiurl
+
+ def sr_for_package(self, project, package):
+ query = "(state/@name='new' or state/@name='review') and " \
+ "(action/source/@project='{project}' or
submit/source/@project='{project}') and " \
+ "(action/source/@package='{package}' or
submit/source/@package='Packafe')".format(project=project, package=package)
+ result = osc.core.search(self.apiurl, request=query)
+ collection = result['request']
+ for root in collection.findall('request'):
+ return root.get('id')
+ return 0
+
+ def create(self, project, package, target, message):
+ currev = osc.core.get_source_rev(self.apiurl, project, package)['rev']
+ print("Creating a request from
{project}/{package}".format(project=project, package=package))
+ query = {'cmd': 'create'}
+ url = osc.core.makeurl(self.apiurl, ['request'], query=query)
+
+ data = '<request type="submit"><submit><source project="{project}"
package="{package}" rev="{rev}"/>' \
+ '<target project="{target}"
package="{package}"/></submit><state
name="new"/><description>{message}</description>' \
+ '</request>'.format(project=project, package=package,
target=target, rev=currev, message=message)
+ osc.core.http_POST(url, data=data)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20220321.48333852/osclib/accept_command.py
new/openSUSE-release-tools-20220323.bd336586/osclib/accept_command.py
--- old/openSUSE-release-tools-20220321.48333852/osclib/accept_command.py
2022-03-21 17:38:05.000000000 +0100
+++ new/openSUSE-release-tools-20220323.bd336586/osclib/accept_command.py
2022-03-23 16:41:56.000000000 +0100
@@ -12,6 +12,7 @@
from osclib.core import attribute_value_load
from osclib.core import source_file_load
from osclib.core import source_file_save
+from osclib.pkglistgen_comments import PkglistComments
from datetime import date
@@ -19,6 +20,7 @@
def __init__(self, api):
self.api = api
self.config = conf.config[self.api.project]
+ self.pkglist_comments = PkglistComments(self.api.apiurl)
def find_new_requests(self, project):
match = f"state/@name='new' and action/target/@project='{project}'"
@@ -133,6 +135,7 @@
self.api.staging_deactivate(project)
+ self.pkglist_comments.check_staging_accept(project,
self.api.project)
self.reset_rebuild_data(project)
if cleanup:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20220321.48333852/osclib/core.py
new/openSUSE-release-tools-20220323.bd336586/osclib/core.py
--- old/openSUSE-release-tools-20220321.48333852/osclib/core.py 2022-03-21
17:38:05.000000000 +0100
+++ new/openSUSE-release-tools-20220323.bd336586/osclib/core.py 2022-03-23
16:41:56.000000000 +0100
@@ -127,8 +127,11 @@
return users
-def package_list(apiurl, project):
- url = makeurl(apiurl, ['source', project], {'expand': 1})
+def package_list(apiurl, project, expand=True):
+ query = {}
+ if expand:
+ query['expand'] = 1
+ url = makeurl(apiurl, ['source', project], query)
root = ET.parse(http_GET(url)).getroot()
packages = []
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20220321.48333852/osclib/pkglistgen_comments.py
new/openSUSE-release-tools-20220323.bd336586/osclib/pkglistgen_comments.py
--- old/openSUSE-release-tools-20220321.48333852/osclib/pkglistgen_comments.py
1970-01-01 01:00:00.000000000 +0100
+++ new/openSUSE-release-tools-20220323.bd336586/osclib/pkglistgen_comments.py
2022-03-23 16:41:56.000000000 +0100
@@ -0,0 +1,277 @@
+import datetime
+import textwrap
+import re
+import tempfile
+import logging
+import os
+import sys
+from lxml import etree as ET
+
+from osclib.comments import CommentAPI
+from osc.core import checkout_package, http_GET, makeurl
+from osc.core import Package
+
+MARKER = 'PackageListDiff'
+
+
+class PkglistComments(object):
+ """Handling staging comments of diffs"""
+
+ def __init__(self, apiurl):
+ self.apiurl = apiurl
+ self.comment = CommentAPI(apiurl)
+
+ def read_summary_file(self, file):
+ ret = dict()
+ with open(file, 'r') as f:
+ for line in f:
+ pkg, group = line.strip().split(':')
+ ret.setdefault(pkg, [])
+ ret[pkg].append(group)
+ return ret
+
+ def write_summary_file(self, file, content):
+ output = []
+ for pkg in sorted(content):
+ for group in sorted(content[pkg]):
+ output.append(f"{pkg}:{group}")
+
+ with open(file, 'w') as f:
+ for line in sorted(output):
+ f.write(line + '\n')
+
+ def calculcate_package_diff(self, old_file, new_file):
+ old_file = self.read_summary_file(old_file)
+ new_file = self.read_summary_file(new_file)
+
+ # remove common part
+ keys = list(old_file.keys())
+ for key in keys:
+ if new_file.get(key, []) == old_file[key]:
+ del new_file[key]
+ del old_file[key]
+
+ if not old_file and not new_file:
+ return None
+
+ removed = dict()
+ for pkg in old_file:
+ old_groups = old_file[pkg]
+ if new_file.get(pkg):
+ continue
+ removekey = ','.join(old_groups)
+ removed.setdefault(removekey, [])
+ removed[removekey].append(pkg)
+
+ report = ''
+ for rm in sorted(removed.keys()):
+ report += f"**Remove from {rm}**\n\n```\n"
+ paragraph = ', '.join(removed[rm])
+ report += "\n".join(textwrap.wrap(paragraph, width=90,
break_long_words=False, break_on_hyphens=False))
+ report += "\n```\n\n"
+
+ moved = dict()
+ for pkg in old_file:
+ old_groups = old_file[pkg]
+ new_groups = new_file.get(pkg)
+ if not new_groups:
+ continue
+ movekey = ','.join(old_groups) + ' to ' + ','.join(new_groups)
+ moved.setdefault(movekey, [])
+ moved[movekey].append(pkg)
+
+ for move in sorted(moved.keys()):
+ report += f"**Move from {move}**\n\n```\n"
+ paragraph = ', '.join(moved[move])
+ report += "\n".join(textwrap.wrap(paragraph, width=90,
break_long_words=False, break_on_hyphens=False))
+ report += "\n```\n\n"
+
+ added = dict()
+ for pkg in new_file:
+ if pkg in old_file:
+ continue
+ addkey = ','.join(new_file[pkg])
+ added.setdefault(addkey, [])
+ added[addkey].append(pkg)
+
+ for group in sorted(added):
+ report += f"**Add to {group}**\n\n```\n"
+ paragraph = ', '.join(added[group])
+ report += "\n".join(textwrap.wrap(paragraph, width=90,
break_long_words=False, break_on_hyphens=False))
+ report += "\n```\n\n"
+
+ return report.strip()
+
+ def handle_package_diff(self, project, old_file, new_file):
+ comments = self.comment.get_comments(project_name=project)
+ comment, _ = self.comment.comment_find(comments, MARKER)
+
+ report = self.calculcate_package_diff(old_file, new_file)
+ if not report:
+ if comment:
+ self.comment.delete(comment['id'])
+ return 0
+ report = self.comment.add_marker(report, MARKER)
+
+ if comment:
+ write_comment = report != comment['comment']
+ else:
+ write_comment = True
+ if write_comment:
+ if comment:
+ self.comment.delete(comment['id'])
+ self.comment.add_comment(project_name=project, comment=report)
+ else:
+ for c in comments.values():
+ if c['parent'] == comment['id']:
+ ct = c['comment']
+ if ct.startswith('ignore ') or ct == 'ignore':
+ print(c)
+ return 0
+ if ct.startswith('approve ') or ct == 'approve':
+ print(c)
+ return 0
+
+ return 1
+
+ def is_approved(self, comment, comments):
+ if not comment:
+ return None
+
+ for c in comments.values():
+ if c['parent'] == comment['id']:
+ ct = c['comment']
+ if ct.startswith('approve ') or ct == 'approve':
+ return c['who']
+ return None
+
+ def parse_title(self, line):
+ m = re.match(r'\*\*Add to (.*)\*\*', line)
+ if m:
+ return {'cmd': 'add', 'to': m.group(1).split(','), 'pkgs': []}
+ m = re.match(r'\*\*Move from (.*) to (.*)\*\*', line)
+ if m:
+ return {'cmd': 'move', 'from': m.group(1).split(','), 'to':
m.group(2).split(','), 'pkgs': []}
+ m = re.match(r'\*\*Remove from (.*)\*\*', line)
+ if m:
+ return {'cmd': 'remove', 'from': m.group(1).split(','), 'pkgs': []}
+ return None
+
+ def parse_sections(self, comment):
+ current_section = None
+ sections = []
+ in_quote = False
+ for line in comment.split('\n'):
+ if line.startswith('**'):
+ if current_section:
+ sections.append(current_section)
+ current_section = self.parse_title(line)
+ continue
+ if line.startswith("```"):
+ in_quote = not in_quote
+ continue
+ if in_quote:
+ for pkg in line.split(','):
+ pkg = pkg.strip()
+ if pkg:
+ current_section['pkgs'].append(pkg)
+ if current_section:
+ sections.append(current_section)
+ return sections
+
+ def apply_move(self, content, section):
+ for pkg in section['pkgs']:
+ pkg_content = content[pkg]
+ for group in section['from']:
+ try:
+ pkg_content.remove(group)
+ except ValueError:
+ logging.error(f"Can't remove {pkg} from {group}, not
there. Mismatch.")
+ sys.exit(1)
+ for group in section['to']:
+ pkg_content.append(group)
+ content[pkg] = pkg_content
+
+ def apply_add(self, content, section):
+ for pkg in section['pkgs']:
+ content.setdefault(pkg, [])
+ content[pkg] += section['to']
+
+ def apply_remove(self, content, section):
+ for pkg in section['pkgs']:
+ pkg_content = content[pkg]
+ for group in section['from']:
+ try:
+ pkg_content.remove(group)
+ except ValueError:
+ logging.error(f"Can't remove {pkg} from {group}, not
there. Mismatch.")
+ sys.exit(1)
+ content[pkg] = pkg_content
+
+ def apply_commands(self, filename, sections):
+ content = self.read_summary_file(filename)
+ for section in sections:
+ if section['cmd'] == 'move':
+ self.apply_move(content, section)
+ elif section['cmd'] == 'add':
+ self.apply_add(content, section)
+ elif section['cmd'] == 'remove':
+ self.apply_remove(content, section)
+ self.write_summary_file(filename, content)
+
+ def format_pkgs(self, pkgs):
+ text = ', '.join(pkgs)
+ return " " + "\n ".join(textwrap.wrap(text, width=68,
break_long_words=False, break_on_hyphens=False)) + "\n\n"
+
+ def format_move(self, section):
+ gfrom = ','.join(section['from'])
+ gto = ','.join(section['to'])
+ text = f" * Move from {gfrom} to {gto}:\n"
+ return text + self.format_pkgs(section['pkgs'])
+
+ def format_add(self, section):
+ gto = ','.join(section['to'])
+ text = f" * Add to {gto}:\n"
+ return text + self.format_pkgs(section['pkgs'])
+
+ def format_remove(self, section):
+ gfrom = ','.join(section['from'])
+ text = f" * Remove from {gfrom}:\n"
+ return text + self.format_pkgs(section['pkgs'])
+
+ def apply_changes(self, filename, sections, approver):
+ text =
"-------------------------------------------------------------------\n"
+ now = datetime.datetime.utcnow()
+ date = now.strftime("%a %b %d %H:%M:%S UTC %Y")
+ url = makeurl(self.apiurl, ['person', approver])
+ root = ET.parse(http_GET(url))
+ realname = root.find('realname').text
+ email = root.find('email').text
+ text += f"{date} - {realname} <{email}>\n\n- Approved changes to
summary-staging.txt\n"
+ for section in sections:
+ if section['cmd'] == 'move':
+ text += self.format_move(section)
+ elif section['cmd'] == 'add':
+ text += self.format_add(section)
+ elif section['cmd'] == 'remove':
+ text += self.format_remove(section)
+ with open(filename + '.new', 'w') as writer:
+ writer.write(text)
+ with open(filename, 'r') as reader:
+ for line in reader:
+ writer.write(line)
+ os.rename(filename + '.new', filename)
+
+ def check_staging_accept(self, project, target):
+ comments = self.comment.get_comments(project_name=project)
+ comment, _ = self.comment.comment_find(comments, MARKER)
+ approver = self.is_approved(comment, comments)
+ if not approver:
+ return
+ sections = self.parse_sections(comment['comment'])
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ checkout_package(self.apiurl, target, '000package-groups',
expand_link=True, outdir=tmpdirname)
+ self.apply_commands(tmpdirname + '/summary-staging.txt', sections)
+ self.apply_changes(tmpdirname + '/package-groups.changes',
sections, approver)
+ package = Package(tmpdirname)
+ package.commit(msg='Approved packagelist changes',
skip_local_service_run=True)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20220321.48333852/pkglistgen/tool.py
new/openSUSE-release-tools-20220323.bd336586/pkglistgen/tool.py
--- old/openSUSE-release-tools-20220321.48333852/pkglistgen/tool.py
2022-03-21 17:38:05.000000000 +0100
+++ new/openSUSE-release-tools-20220323.bd336586/pkglistgen/tool.py
2022-03-23 16:41:56.000000000 +0100
@@ -7,7 +7,6 @@
import shutil
import subprocess
import yaml
-import textwrap
from lxml import etree as ET
@@ -23,7 +22,7 @@
from osclib.core import repository_path_expand
from osclib.core import repository_arch_state
from osclib.cache_manager import CacheManager
-from osclib.comments import CommentAPI
+from osclib.pkglistgen_comments import PkglistComments
from urllib.parse import urlparse
@@ -33,7 +32,6 @@
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product'
-MARKER = 'PackageListDiff'
# share header cache with repochecker
CACHEDIR = CacheManager.directory('repository-meta')
@@ -48,7 +46,7 @@
def __init__(self):
ToolBase.ToolBase.__init__(self)
self.logger = logging.getLogger(__name__)
- self.comment = CommentAPI(self.apiurl)
+ self.comment = PkglistComments(self.apiurl)
self.reset()
def reset(self):
@@ -508,106 +506,6 @@
print('%endif', file=output)
output.flush()
- def read_summary_file(self, file):
- ret = dict()
- with open(file, 'r') as f:
- for line in f:
- pkg, group = line.strip().split(':')
- ret.setdefault(pkg, [])
- ret[pkg].append(group)
- return ret
-
- def calculcate_package_diff(self, old_file, new_file):
- old_file = self.read_summary_file(old_file)
- new_file = self.read_summary_file(new_file)
-
- # remove common part
- keys = list(old_file.keys())
- for key in keys:
- if new_file.get(key, []) == old_file[key]:
- del new_file[key]
- del old_file[key]
-
- if not old_file and not new_file:
- return None
-
- removed = dict()
- for pkg in old_file:
- old_groups = old_file[pkg]
- if new_file.get(pkg):
- continue
- removekey = ','.join(old_groups)
- removed.setdefault(removekey, [])
- removed[removekey].append(pkg)
-
- report = ''
- for rm in sorted(removed.keys()):
- report += f"**Remove from {rm}**\n\n```\n"
- paragraph = ', '.join(removed[rm])
- report += "\n".join(textwrap.wrap(paragraph, width=90,
break_long_words=False, break_on_hyphens=False))
- report += "\n```\n\n"
-
- moved = dict()
- for pkg in old_file:
- old_groups = old_file[pkg]
- new_groups = new_file.get(pkg)
- if not new_groups:
- continue
- movekey = ','.join(old_groups) + ' to ' + ','.join(new_groups)
- moved.setdefault(movekey, [])
- moved[movekey].append(pkg)
-
- for move in sorted(moved.keys()):
- report += f"**Move from {move}**\n\n```\n"
- paragraph = ', '.join(moved[move])
- report += "\n".join(textwrap.wrap(paragraph, width=90,
break_long_words=False, break_on_hyphens=False))
- report += "\n```\n\n"
-
- added = dict()
- for pkg in new_file:
- if pkg in old_file:
- continue
- addkey = ','.join(new_file[pkg])
- added.setdefault(addkey, [])
- added[addkey].append(pkg)
-
- for group in sorted(added):
- report += f"**Add to {group}**\n\n```\n"
- paragraph = ', '.join(added[group])
- report += "\n".join(textwrap.wrap(paragraph, width=90,
break_long_words=False, break_on_hyphens=False))
- report += "\n```\n\n"
-
- return report.strip()
-
- def handle_package_diff(self, project, old_file, new_file):
- comments = self.comment.get_comments(project_name=project)
- comment, _ = self.comment.comment_find(comments, MARKER)
-
- report = self.calculcate_package_diff(old_file, new_file)
- if not report:
- if comment:
- self.comment.delete(comment['id'])
- return 0
- report = self.comment.add_marker(report, MARKER)
-
- if comment:
- write_comment = report != comment['comment']
- else:
- write_comment = True
- if write_comment:
- if comment:
- self.comment.delete(comment['id'])
- self.comment.add_comment(project_name=project, comment=report)
- else:
- for c in comments.values():
- if c['parent'] == comment['id']:
- ct = c['comment']
- if ct.startswith('ignore ') or ct == 'ignore':
- print(c)
- return 0
-
- return 1
-
def solve_project(self, ignore_unresolvable=False,
ignore_recommended=False, locale=None, locales_from=None):
self.load_all_groups()
if not self.output:
@@ -766,8 +664,6 @@
checkout_package(api.apiurl, project, package, expand_link=True,
prj_dir=cache_dir, outdir=os.path.join(cache_dir,
package))
- # print('RET', self.handle_package_diff(project,
f"{group_dir}/summary-staging.txt", f"{product_dir}/summary-staging.txt"))
-
file_utils.unlink_all_except(release_dir, ['weakremovers.inc'])
if not only_release_packages:
file_utils.unlink_all_except(product_dir)
@@ -860,4 +756,4 @@
self.commit_package(product_dir)
if os.path.isfile(reference_summary):
- return self.handle_package_diff(project, reference_summary,
summary_file)
+ return self.comment.handle_package_diff(project,
reference_summary, summary_file)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20220321.48333852/staging-installcheck.py
new/openSUSE-release-tools-20220323.bd336586/staging-installcheck.py
--- old/openSUSE-release-tools-20220321.48333852/staging-installcheck.py
2022-03-21 17:38:05.000000000 +0100
+++ new/openSUSE-release-tools-20220323.bd336586/staging-installcheck.py
2022-03-23 16:41:56.000000000 +0100
@@ -11,7 +11,6 @@
import osc.core
import yaml
from lxml import etree as ET
-from osc import conf
from osclib.comments import CommentAPI
from osclib.conf import Config
@@ -19,6 +18,7 @@
from osclib.core import (builddepinfo, depends_on, duplicated_binaries_in_repo,
fileinfo_ext_all, repository_arch_state,
repository_path_expand, target_archs)
+
from osclib.repochecks import installcheck, mirror
from osclib.stagingapi import StagingAPI
@@ -29,22 +29,21 @@
class InstallChecker(object):
def __init__(self, api, config):
self.api = api
- self.config = conf.config[api.project]
self.logger = logging.getLogger('InstallChecker')
self.commentapi = CommentAPI(api.apiurl)
- self.arch_whitelist = self.config.get('repo_checker-arch-whitelist')
+ self.arch_whitelist = config.get('repo_checker-arch-whitelist')
if self.arch_whitelist:
self.arch_whitelist = set(self.arch_whitelist.split(' '))
- self.ring_whitelist =
set(self.config.get('repo_checker-binary-whitelist-ring', '').split(' '))
+ self.ring_whitelist =
set(config.get('repo_checker-binary-whitelist-ring', '').split(' '))
- self.cycle_packages = self.config.get('repo_checker-allowed-in-cycles')
+ self.cycle_packages = config.get('repo_checker-allowed-in-cycles')
self.calculate_allowed_cycles()
- self.ignore_duplicated =
set(self.config.get('installcheck-ignore-duplicated-binaries', '').split(' '))
- self.ignore_conflicts =
set(self.config.get('installcheck-ignore-conflicts', '').split(' '))
- self.ignore_deletes =
str2bool(self.config.get('installcheck-ignore-deletes', 'False'))
+ self.ignore_duplicated =
set(config.get('installcheck-ignore-duplicated-binaries', '').split(' '))
+ self.ignore_conflicts =
set(config.get('installcheck-ignore-conflicts', '').split(' '))
+ self.ignore_deletes =
str2bool(config.get('installcheck-ignore-deletes', 'False'))
def check_required_by(self, fileinfo, provides, requiredby,
built_binaries, comments):
if requiredby.get('name') in built_binaries:
@@ -374,7 +373,7 @@
osc.conf.config['debug'] = args.debug
apiurl = osc.conf.config['apiurl']
- config = Config(apiurl, args.project)
+ config = Config.get(apiurl, args.project)
api = StagingAPI(apiurl, args.project)
staging_report = InstallChecker(api, config)
@@ -383,13 +382,11 @@
else:
logging.basicConfig(level=logging.INFO)
- result = True
if args.staging:
- result = staging_report.staging(api.prj_from_short(args.staging),
force=True)
+ if not staging_report.staging(api.prj_from_short(args.staging),
force=True):
+ sys.exit(1)
else:
for staging in api.get_staging_projects():
if api.is_adi_project(staging):
- result = staging_report.staging(staging) and result
-
- if not result:
- logging.error("Found problem")
+ staging_report.staging(staging)
+ sys.exit(0)
++++++ openSUSE-release-tools.obsinfo ++++++
--- /var/tmp/diff_new_pack.PVnDVa/_old 2022-03-23 20:21:11.118558041 +0100
+++ /var/tmp/diff_new_pack.PVnDVa/_new 2022-03-23 20:21:11.126558046 +0100
@@ -1,5 +1,5 @@
name: openSUSE-release-tools
-version: 20220321.48333852
-mtime: 1647880685
-commit: 4833385205225ad2ef4d4686ac7f2fc2d54a78cc
+version: 20220323.bd336586
+mtime: 1648050116
+commit: bd3365863f090bfe86f03dd73b483431c9ed33f3
