Hello community,
here is the log from the commit of package openSUSE-release-tools for
openSUSE:Factory checked in at 2018-01-02 16:35:44
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old)
and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openSUSE-release-tools"
Tue Jan 2 16:35:44 2018 rev:36 rq:561094 version:20180102.98a659c
Changes:
--------
---
/work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes
2017-12-29 18:51:04.442062391 +0100
+++
/work/SRC/openSUSE:Factory/.openSUSE-release-tools.new/openSUSE-release-tools.changes
2018-01-02 16:35:50.539375711 +0100
@@ -1,0 +2,17 @@
+Tue Jan 02 10:50:33 UTC 2018 - opensuse-releaset...@opensuse.org
+
+- Update to version 20180102.98a659c:
+ * manager42: ignore meta packages
+ * manager42: refactor
+ * manager42: don't turn dropped packages into fork
+ * manager42: warn about inconsistent packages
+ * reochecker: avoid accessing empty status
+ * repo_checker: only update comment if published or message changed.
+ * ReviewBot: comment_write(): provide info_extra_identical parameter.
+ * fcc_submitter: handle multibuild package in build_succeeded package list
+ * check_duplicate_binaries: better yaml
+ * pkglistgen: migrate bash scripts to python.
+ * osclib/stagingapi: provide is_staging_bootstrapped() from request_splitter.
+ * ToolBase: provide -d alias for --debug for consistency.
+
+-------------------------------------------------------------------
Old:
----
openSUSE-release-tools-20171226.96a3eec.obscpio
New:
----
openSUSE-release-tools-20180102.98a659c.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ openSUSE-release-tools.spec ++++++
--- /var/tmp/diff_new_pack.qsXQKW/_old 2018-01-02 16:35:51.599022346 +0100
+++ /var/tmp/diff_new_pack.qsXQKW/_new 2018-01-02 16:35:51.603021012 +0100
@@ -1,7 +1,7 @@
#
# spec file for package openSUSE-release-tools
#
-# Copyright (c) 2017 SUSE LINUX GmbH, Nuernberg, Germany.
+# Copyright (c) 2018 SUSE LINUX GmbH, Nuernberg, Germany.
#
# All modifications and additions to the file contributed by third parties
# remain the property of their copyright owners, unless otherwise agreed
@@ -20,7 +20,7 @@
%define source_dir osc-plugin-factory
%define announcer_filename factory-package-news
Name: openSUSE-release-tools
-Version: 20171226.96a3eec
+Version: 20180102.98a659c
Release: 0
Summary: Tools to aid in staging and release work for openSUSE/SUSE
License: GPL-2.0+ and MIT
@@ -483,11 +483,6 @@
%files pkglistgen
%defattr(-,root,root,-)
%{_bindir}/osrt-pkglistgen
-%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0
-%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0-all
-%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Rings
-%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Staging
-%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh
%{_unitdir}/osrt-pkglistgen@.service
%{_unitdir}/osrt-pkglistgen@.timer
++++++ _servicedata ++++++
--- /var/tmp/diff_new_pack.qsXQKW/_old 2018-01-02 16:35:51.651005011 +0100
+++ /var/tmp/diff_new_pack.qsXQKW/_new 2018-01-02 16:35:51.651005011 +0100
@@ -1,6 +1,6 @@
<servicedata>
<service name="tar_scm">
<param
name="url">https://github.com/openSUSE/osc-plugin-factory.git</param>
- <param
name="changesrevision">96a3eec425d49f995a85e000da4a8799b5a6a461</param>
+ <param
name="changesrevision">98a659c3fab8dc036f9945854eae93265d695bed</param>
</service>
</servicedata>
++++++ openSUSE-release-tools-20171226.96a3eec.obscpio ->
openSUSE-release-tools-20180102.98a659c.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/Makefile
new/openSUSE-release-tools-20180102.98a659c/Makefile
--- old/openSUSE-release-tools-20171226.96a3eec/Makefile 2017-12-26
12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/Makefile 2018-01-02
11:39:25.000000000 +0100
@@ -9,7 +9,7 @@
leaper.py \
manager_42.py \
metrics.py \
- pkglistgen.sh \
+ pkglistgen.py \
repo_checker.py \
suppkg_rebuild.py \
totest-manager.py \
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/ReviewBot.py
new/openSUSE-release-tools-20180102.98a659c/ReviewBot.py
--- old/openSUSE-release-tools-20171226.96a3eec/ReviewBot.py 2017-12-26
12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/ReviewBot.py 2018-01-02
11:39:25.000000000 +0100
@@ -436,13 +436,15 @@
def comment_write(self, state='done', result=None, project=None,
package=None,
request=None, message=None, identical=False,
only_replace=False,
- info_extra=None):
+ info_extra=None, info_extra_identical=True):
"""Write comment if not similar to previous comment and replace old
one.
The state, result, and info_extra (dict) are combined to create the
info
that is passed to CommentAPI methods for creating a marker and finding
previous comments. self.bot_name, which defaults to class, will be used
- as the primary matching key.
+ as the primary matching key. When info_extra_identical is set to False
+ info_extra will not be included when finding previous comments to
+ compare message against.
A comment from the same bot will be replaced when a new comment is
written. The only_replace flag will restrict to only writing a comment
@@ -482,15 +484,24 @@
message = '\n\n'.join(self.comment_handler.lines)
info = {'state': state, 'result': result}
- if info_extra:
+ if info_extra and info_extra_identical:
info.update(info_extra)
- message = self.comment_api.add_marker(message, self.bot_name, info)
- message = self.comment_api.truncate(message.strip())
comments = self.comment_api.get_comments(**kwargs)
comment, _ = self.comment_api.comment_find(comments, self.bot_name,
info)
+
+ if info_extra and not info_extra_identical:
+ # Add info_extra once comment has already been matched.
+ info.update(info_extra)
+
+ message = self.comment_api.add_marker(message, self.bot_name, info)
+ message = self.comment_api.truncate(message.strip())
+
if (comment is not None and
- ((identical and comment['comment'] == message) or
+ ((identical and
+ # Remove marker from comments since handled during
comment_find().
+ self.comment_api.remove_marker(comment['comment']) ==
+ self.comment_api.remove_marker(message)) or
(not identical and comment['comment'].count('\n') ==
message.count('\n')))
):
# Assume same state/result and number of lines in message is
duplicate.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openSUSE-release-tools-20171226.96a3eec/ToolBase.py
new/openSUSE-release-tools-20180102.98a659c/ToolBase.py
--- old/openSUSE-release-tools-20171226.96a3eec/ToolBase.py 2017-12-26
12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/ToolBase.py 2018-01-02
11:39:25.000000000 +0100
@@ -140,7 +140,7 @@
parser = cmdln.Cmdln.get_optparser(self)
parser.add_option("--apiurl", '-A', metavar="URL", help="api url")
parser.add_option("--dry", action="store_true", help="dry run")
- parser.add_option("--debug", action="store_true", help="debug output")
+ parser.add_option("-d", "--debug", action="store_true", help="debug
output")
parser.add_option("--osc-debug", action="store_true", help="osc debug
output")
parser.add_option("--verbose", action="store_true", help="verbose")
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/dist/package/openSUSE-release-tools.spec
new/openSUSE-release-tools-20180102.98a659c/dist/package/openSUSE-release-tools.spec
---
old/openSUSE-release-tools-20171226.96a3eec/dist/package/openSUSE-release-tools.spec
2017-12-26 12:36:26.000000000 +0100
+++
new/openSUSE-release-tools-20180102.98a659c/dist/package/openSUSE-release-tools.spec
2018-01-02 11:39:25.000000000 +0100
@@ -483,11 +483,6 @@
%files pkglistgen
%defattr(-,root,root,-)
%{_bindir}/osrt-pkglistgen
-%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0
-%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0-all
-%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Rings
-%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Staging
-%{_bindir}/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh
%{_unitdir}/osrt-pkglistgen@.service
%{_unitdir}/osrt-pkglistgen@.timer
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/fcc_submitter.py
new/openSUSE-release-tools-20180102.98a659c/fcc_submitter.py
--- old/openSUSE-release-tools-20171226.96a3eec/fcc_submitter.py
2017-12-26 12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/fcc_submitter.py
2018-01-02 11:39:25.000000000 +0100
@@ -184,10 +184,25 @@
root = ET.fromstring(''.join(f))
#print ET.dump(root)
+ failed_multibuild_pacs = []
pacs = []
for node in root.findall('result'):
if node.get('repository') == 'pure_42' and node.get('arch') ==
'x86_64':
for pacnode in node.findall('status'):
+ if ':' in pacnode.get('package'):
+ mainpac = pacnode.get('package').split(':')[0]
+ if pacnode.get('code') not in ['succeeded',
'excluded']:
+
failed_multibuild_pacs.append(pacnode.get('package'))
+ if mainpac not in failed_multibuild_pacs:
+ failed_multibuild_pacs.append(mainpac)
+ if mainpac in pacs:
+ pacs.remove(mainpac)
+ else:
+ if mainpac in failed_multibuild_pacs:
+
failed_multibuild_pacs.append(pacnode.get('package'))
+ elif mainpac not in pacs:
+ pacs.append(mainpac)
+ continue
if pacnode.get('code') == 'succeeded':
pacs.append(pacnode.get('package'))
else:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/manager_42.py
new/openSUSE-release-tools-20180102.98a659c/manager_42.py
--- old/openSUSE-release-tools-20171226.96a3eec/manager_42.py 2017-12-26
12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/manager_42.py 2018-01-02
11:39:25.000000000 +0100
@@ -46,7 +46,15 @@
class Manager42(object):
config_defaults = {
+ 'ignored_packages' : [
+ '00Meta',
+ '00aggregates',
+ '000product',
+ '000package-groups',
+ '000release-packages',
+ ],
'project_preference_order' : [],
+ 'drop_if_vanished_from' : [],
'from_prj' : 'openSUSE:Leap:42.3',
'factory' : 'openSUSE:Factory',
}
@@ -61,7 +69,7 @@
self.fill_package_meta()
self.packages = dict()
for project in [self.config.from_prj] +
self.config.project_preference_order:
- self.packages[project] = self.get_source_packages(project)
+ self._fill_package_list(project)
# FIXME: add to ToolBase and rebase Manager42 on that
def _load_config(self, handle = None):
@@ -81,6 +89,9 @@
packages.add(title[3:].split(' ')[0])
return sorted(packages)
+ def all_packages(self):
+ return self.packages[self.config.from_prj]
+
def parse_lookup(self, project):
self.lookup_changes = 0
self.lookup = {}
@@ -152,11 +163,9 @@
['source', project, package], opts))
- def crawl(self, given_packages = None):
+ def crawl(self, packages):
"""Main method of the class that runs the crawler."""
- packages = given_packages or self.packages[self.config.from_prj]
-
for package in sorted(packages):
try:
self.check_one_package(package)
@@ -171,6 +180,18 @@
if self.lookup_changes:
self.store_lookup()
+ def get_inconsistent(self):
+ known = set(self.lookup.keys())
+ stale = known - set(self.packages[self.config.from_prj])
+ unknown = set(self.packages[self.config.from_prj]) - known
+
+ if (stale):
+ logger.info("stale packages: %s", ', '.join(stale))
+ if (unknown):
+ logger.info("unknown packages: %s", ', '.join(unknown))
+
+ return (stale|unknown)
+
def get_package_history(self, project, package, deleted = False):
try:
query = {}
@@ -183,9 +204,20 @@
return None
raise
- def check_source_in_project(self, project, package, verifymd5,
deleted=False):
+
+ def _is_ignored(self, project, package):
+ if package in self.config.ignored_packages:
+ logger.debug("%s in ignore list", package)
+ return True
+ return False
+
+ def _fill_package_list(self, project):
if project not in self.packages:
- self.packages[project] = self.get_source_packages(project)
+ self.packages[project] = [ p for p in
self.get_source_packages(project) if not self._is_ignored(project, p) ]
+
+ def check_source_in_project(self, project, package, verifymd5,
deleted=False):
+
+ self._fill_package_list(project)
if not deleted and not package in self.packages[project]:
return None, None
@@ -289,6 +321,8 @@
if not foundit:
if lproject == 'FORK':
logger.debug("{}: lookup is correctly marked as
fork".format(package))
+ elif lproject in self.config.drop_if_vanished_from:
+ logger.info('{} dropped from {}'.format(package, lproject))
else:
logger.info('{} is a fork (was {})'.format(package, lproject))
self.lookup[package] = 'FORK'
@@ -318,9 +352,15 @@
osc.conf.config['debug'] = args.debug
uc = Manager42(caching = args.cache_requests, configfh = args.config )
- given_packages = args.packages
- if not args.all and not given_packages:
- given_packages = uc.latest_packages()
+ given_packages = set(args.packages)
+ if args.all:
+ given_packages = set(uc.all_packages())
+ elif not given_packages:
+ given_packages = set(uc.latest_packages())
+
+ if args.check_inconsistent:
+ given_packages |= uc.get_inconsistent()
+
if args.force:
uc.force = True
uc.crawl(given_packages)
@@ -342,6 +382,8 @@
help='don\'t take previous lookup information into
consideration')
parser.add_argument('--cache-requests', action='store_true', default=False,
help='cache GET requests. Not recommended for daily
use.')
+ parser.add_argument('--check-inconsistent', action='store_true',
default=False,
+ help='also check insonsistent lookup entries')
parser.add_argument("packages", nargs='*', help="packages to check")
args = parser.parse_args()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/osclib/check_duplicate_binaries_command.py
new/openSUSE-release-tools-20180102.98a659c/osclib/check_duplicate_binaries_command.py
---
old/openSUSE-release-tools-20171226.96a3eec/osclib/check_duplicate_binaries_command.py
2017-12-26 12:36:26.000000000 +0100
+++
new/openSUSE-release-tools-20180102.98a659c/osclib/check_duplicate_binaries_command.py
2018-01-02 11:39:25.000000000 +0100
@@ -60,14 +60,19 @@
binaries[arch][name] = package
+ # convert sets to lists for readable yaml
+ for arch in duplicates.keys():
+ for name in duplicates[arch].keys():
+ duplicates[arch][name] = list(duplicates[arch][name])
+
+ current = yaml.dump(duplicates, default_flow_style=False)
if save:
args = ['{}:Staging'.format(self.api.project), 'dashboard',
'duplicate_binaries']
previous = self.api.load_file_content(*args)
- current = yaml.dump(duplicates, default_flow_style=False)
if current != previous:
args.append(current)
self.api.save_file_content(*args)
else:
- pprint(duplicates)
+ print(current)
# vim: sw=4 et
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/osclib/comments.py
new/openSUSE-release-tools-20180102.98a659c/osclib/comments.py
--- old/openSUSE-release-tools-20171226.96a3eec/osclib/comments.py
2017-12-26 12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/osclib/comments.py
2018-01-02 11:39:25.000000000 +0100
@@ -123,6 +123,12 @@
marker = '<!-- {}{} -->'.format(bot, ' ' + ' '.join(infos) if info
else '')
return marker + '\n\n' + comment
+ def remove_marker(self, comment):
+ if comment.startswith('<!--'):
+ comment = ''.join(comment.splitlines(True)[1:]).strip()
+
+ return comment
+
def add_comment(self, request_id=None, project_name=None,
package_name=None, comment=None, parent_id=None):
"""Add a comment in an object in OBS.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/osclib/conf.py
new/openSUSE-release-tools-20180102.98a659c/osclib/conf.py
--- old/openSUSE-release-tools-20171226.96a3eec/osclib/conf.py 2017-12-26
12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/osclib/conf.py 2018-01-02
11:39:25.000000000 +0100
@@ -69,6 +69,11 @@
# check_source.py
# review-team optionally added by leaper.py.
'repo-checker': 'repo-checker',
+ 'pkglistgen-archs': 'x86_64',
+ 'pkglistgen-archs-ports': 'aarch64',
+ 'pkglistgen-locals-from': 'openSUSE-product',
+ 'pkglistgen-include-suggested': '1',
+ 'pkglistgen-delete-kiwis': 'openSUSE-ftp-ftp-x86_64.kiwi
openSUSE-cd-mini-x86_64.kiwi',
},
r'SUSE:(?P<project>SLE-15.*$)': {
'staging': 'SUSE:%(project)s:Staging',
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/osclib/request_splitter.py
new/openSUSE-release-tools-20180102.98a659c/osclib/request_splitter.py
--- old/openSUSE-release-tools-20171226.96a3eec/osclib/request_splitter.py
2017-12-26 12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/osclib/request_splitter.py
2018-01-02 11:39:25.000000000 +0100
@@ -176,15 +176,6 @@
return '00'
return '__'.join(key)
- def is_staging_bootstrapped(self, project):
- if self.api.rings:
- # Determine if staging is bootstrapped.
- meta = self.api.get_prj_meta(project)
- xpath = 'link[@project="{}"]'.format(self.api.rings[0])
- return meta.find(xpath) is not None
-
- return False
-
def is_staging_mergeable(self, status, pseudometa):
return len(pseudometa['requests']) > 0 and 'splitter_info' in
pseudometa
@@ -238,7 +229,7 @@
for staging in stagings:
project = self.api.prj_from_short(staging)
status, pseudometa = self.staging_status_load(project)
- bootstrapped = self.is_staging_bootstrapped(project)
+ bootstrapped = self.api.is_staging_bootstrapped(project)
# Store information about staging.
self.stagings[staging] = {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/osclib/stagingapi.py
new/openSUSE-release-tools-20180102.98a659c/osclib/stagingapi.py
--- old/openSUSE-release-tools-20171226.96a3eec/osclib/stagingapi.py
2017-12-26 12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/osclib/stagingapi.py
2018-01-02 11:39:25.000000000 +0100
@@ -1811,3 +1811,12 @@
break_long_words=False)
return None
+
+ def is_staging_bootstrapped(self, project):
+ if self.rings:
+ # Determine if staging is bootstrapped.
+ meta = self.get_prj_meta(project)
+ xpath = 'link[@project="{}"]'.format(self.rings[0])
+ return meta.find(xpath) is not None
+
+ return False
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/pkglistgen.py
new/openSUSE-release-tools-20180102.98a659c/pkglistgen.py
--- old/openSUSE-release-tools-20171226.96a3eec/pkglistgen.py 2017-12-26
12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/pkglistgen.py 2018-01-02
11:39:25.000000000 +0100
@@ -24,13 +24,22 @@
# TODO: solve all devel packages to include
from __future__ import print_function
+import copy
from lxml import etree as ET
from collections import namedtuple
import sys
import cmdln
import logging
import urllib2
-import osc.core
+from osc.core import checkout_package
+from osc.core import http_GET
+from osc.core import makeurl
+from osc.core import Package
+from osc.core import show_results_meta
+from osc.core import undelete_package
+from osc import conf
+from osclib.conf import Config
+from osclib.stagingapi import StagingAPI
import glob
import solv
from pprint import pprint, pformat
@@ -44,6 +53,7 @@
import gzip
import tempfile
import random
+import shutil
import string
import ToolBase
@@ -55,6 +65,7 @@
ARCHITECTURES = ['x86_64', 'ppc64le', 's390x', 'aarch64']
DEFAULT_REPOS = ("openSUSE:Factory/standard")
+PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product'
class Group(object):
@@ -627,6 +638,7 @@
fh.write(" \n")
class CommandLineInterface(ToolBase.CommandLineInterface):
+ SCOPES = ['all', 'target', 'rings', 'staging', 'ports']
def __init__(self, *args, **kwargs):
ToolBase.CommandLineInterface.__init__(self, args, kwargs)
@@ -929,6 +941,233 @@
self.tool._collect_unsorted_packages(modules)
self.tool._write_all_groups()
+ @cmdln.option('-f', '--force', action='store_true', help='continue even if
build is in progress')
+ @cmdln.option('-p', '--project', help='target project')
+ @cmdln.option('-s', '--scope', default='all', help='scope on which to
operate ({})'.format(', '.join(SCOPES)))
+ def do_update_and_solve(self, subcmd, opts):
+ """${cmd_name}: update and solve for given scope
+
+ ${cmd_usage}
+ ${cmd_option_list}
+ """
+
+ if not opts.project:
+ raise ValueError('project is required')
+ if opts.scope not in self.SCOPES:
+ raise ValueError('scope must be one of: {}'.format(',
'.join(self.SCOPES)))
+
+ if opts.scope == 'all':
+ for scope in self.SCOPES[1:]:
+ opts.scope = scope
+ self.do_update_and_solve(subcmd, copy.deepcopy(opts))
+ return
+
+ # Store target project as opts.project will contain subprojects.
+ target_project = opts.project
+
+ config = Config(target_project)
+ apiurl = conf.config['apiurl']
+ api = StagingAPI(apiurl, target_project)
+ config.apply_remote(api)
+
+ target_config = conf.config[target_project]
+ archs_key = 'pkglistgen-archs' if opts.scope != 'ports' else
'pkglistgen-archs-ports'
+ if archs_key in target_config:
+ self.options.architectures = target_config.get(archs_key).split('
')
+ main_repo = target_config['main-repo']
+
+ if opts.scope == 'target':
+ self.options.repos = ['/'.join([target_project, main_repo])]
+ self.update_and_solve_target(apiurl, target_project,
target_config, main_repo, opts)
+ return
+ elif opts.scope == 'ports':
+ # TODO Continue supporting #1297, but should be abstracted.
+ main_repo = 'ports'
+ opts.project += ':Ports'
+ self.options.repos = ['/'.join([opts.project, main_repo])]
+ self.update_and_solve_target(apiurl, target_project,
target_config, main_repo, opts)
+ return
+ elif opts.scope == 'rings':
+ opts.project = api.rings[1]
+ self.options.repos = [
+ '/'.join([api.rings[1], main_repo]),
+ '/'.join([api.rings[0], main_repo]),
+ ]
+ self.update_and_solve_target(apiurl, target_project,
target_config, main_repo, opts)
+
+ opts.project = api.rings[2]
+ self.options.repos.insert(0, '/'.join([api.rings[2], main_repo]))
+ self.update_and_solve_target(apiurl, target_project,
target_config, main_repo, opts, skip_release=True)
+ return
+ elif opts.scope == 'staging':
+ letters = api.get_staging_projects_short()
+ for letter in letters:
+ opts.project = api.prj_from_short(letter)
+ self.options.repos = ['/'.join([opts.project, main_repo])]
+
+ if not api.is_staging_bootstrapped(opts.project):
+ self.options.repos.append('/'.join([opts.project,
'bootstrap_copy']))
+
+ # DVD project first since it depends on main.
+ if api.rings:
+ opts_dvd = copy.deepcopy(opts)
+ opts.project += ':DVD'
+ self.options.repos.insert(0, '/'.join([opts.project,
main_repo]))
+ self.update_and_solve_target(apiurl, target_project,
target_config, main_repo, opts_dvd, skip_release=True)
+
+ self.update_and_solve_target(apiurl, target_project,
target_config, main_repo, opts)
+ return
+
+ def update_and_solve_target(self, apiurl, target_project, target_config,
main_repo, opts,
+ skip_release=False):
+ group = target_config.get('pkglistgen-group', '000package-groups')
+ product = target_config.get('pkglistgen-product', '000product')
+ release = target_config.get('pkglistgen-release',
'000release-packages')
+
+ url = makeurl(apiurl, ['source', opts.project])
+ packages = ET.parse(http_GET(url)).getroot()
+ if packages.find('entry[@name="{}"]'.format(product)) is None:
+ undelete_package(apiurl, opts.project, product, 'revive')
+ # TODO disable build.
+ print('{} undeleted, skip dvd until next cycle'.format(product))
+ return
+ elif not opts.force:
+ root = ET.fromstringlist(show_results_meta(apiurl, opts.project,
product,
+ repository=[main_repo],
multibuild=True))
+ if len(root.xpath('result[@state="building"]')) or
len(root.xpath('result[@state="dirty"]')):
+ print('{}/{} build in progress'.format(opts.project, product))
+ return
+
+ checkout_list = [group, product]
+ if not skip_release:
+ checkout_list.append(release)
+
+ if packages.find('entry[@name="{}"]'.format(release)) is None:
+ undelete_package(apiurl, opts.project, product, 'revive')
+ print('{} undeleted, skip dvd until next
cycle'.format(release))
+ return
+
+ # Cache dir specific to hostname and project.
+ host = urlparse.urlparse(apiurl).hostname
+ cache_dir = os.environ.get('XDG_CACHE_HOME',
os.path.expanduser('~/.cache'))
+ cache_dir = os.path.join(cache_dir, 'opensuse-packagelists', host,
opts.project)
+
+ if os.path.exists(cache_dir):
+ shutil.rmtree(cache_dir)
+ os.makedirs(cache_dir)
+
+ group_dir = os.path.join(cache_dir, group)
+ product_dir = os.path.join(cache_dir, product)
+ release_dir = os.path.join(cache_dir, release)
+
+ for package in checkout_list:
+ checkout_package(apiurl, opts.project, package, expand_link=True,
prj_dir=cache_dir)
+
+ if not skip_release:
+ self.unlink_all_except(release_dir)
+ self.unlink_all_except(product_dir)
+ self.copy_directory_contents(group_dir, product_dir,
+ ['supportstatus.txt', 'groups.yml',
'package-groups.changes'])
+ self.change_extension(product_dir, '.spec.in', '.spec')
+
+ self.options.output_dir = product_dir
+ self.postoptparse()
+
+ self.do_update('update', opts)
+
+ opts.ignore_recommended =
bool(target_config.get('pkglistgen-include-recommended'))
+ opts.include_suggested =
bool(target_config.get('pkglistgen-include-suggested'))
+ opts.locales_from = target_config.get('pkglistgen-locals-from')
+ self.do_solve('solve', opts)
+
+ delete_products = target_config.get('pkglistgen-delete-products',
'').split(' ')
+ self.unlink_list(product_dir, delete_products)
+
+ for product_file in glob.glob(os.path.join(product_dir, '*.product')):
+ print(subprocess.check_output(
+ [PRODUCT_SERVICE, product_file, product_dir, opts.project]))
+
+ delete_kiwis = target_config.get('pkglistgen-delete-kiwis',
'').split(' ')
+ self.unlink_list(product_dir, delete_kiwis)
+
+ spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
+ if skip_release:
+ self.unlink_list(None, spec_files)
+ else:
+ self.move_list(spec_files, release_dir)
+
+ self.multibuild_from_glob(product_dir, '*.kiwi')
+ self.build_stub(product_dir, 'kiwi')
+ self.commit_package(product_dir)
+
+ if not skip_release:
+ self.multibuild_from_glob(release_dir, '*.spec')
+ self.build_stub(release_dir, 'spec')
+ self.commit_package(release_dir)
+
+ def move_list(self, file_list, destination):
+ for name in file_list:
+ os.rename(name, os.path.join(destination, os.path.basename(name)))
+
+ def unlink_list(self, path, names):
+ for name in names:
+ if path is None:
+ name_path = name
+ else:
+ name_path = os.path.join(path, name)
+
+ if os.path.isfile(name_path):
+ os.unlink(name_path)
+
+ def unlink_all_except(self, path, ignore_list=['_service'],
ignore_hidden=True):
+ for name in os.listdir(path):
+ if name in ignore_list or (ignore_hidden and name.startswith('.')):
+ continue
+
+ name_path = os.path.join(path, name)
+ if os.path.isfile(name_path):
+ os.unlink(name_path)
+
+ def copy_directory_contents(self, source, destination, ignore_list=[]):
+ for name in os.listdir(source):
+ name_path = os.path.join(source, name)
+ if name in ignore_list or not os.path.isfile(name_path):
+ continue
+
+ shutil.copy(name_path, os.path.join(destination, name))
+
+ def change_extension(self, path, original, final):
+ for name in glob.glob(os.path.join(path, '*{}'.format(original))):
+ # Assumes the extension is only found at the end.
+ os.rename(name, name.replace(original, final))
+
+ def multibuild_from_glob(self, destination, pathname):
+ root = ET.Element('multibuild')
+ for name in glob.glob(os.path.join(destination, pathname)):
+ package = ET.SubElement(root, 'package')
+ package.text = os.path.splitext(os.path.basename(name))[0]
+
+ with open(os.path.join(destination, '_multibuild'), 'w+b') as f:
+ f.write(ET.tostring(root, pretty_print=True))
+
+ def build_stub(self, destination, extension):
+ f = file(os.path.join(destination, '.'.join(['stub', extension])),
'w+')
+ f.write('# prevent building single {} files twice\n'.format(extension))
+ f.write('Name: stub\n')
+ f.write('Version: 0.0\n')
+ f.close()
+
+ def commit_package(self, path):
+ package = Package(path)
+ if self.options.dry:
+ for i in package.get_diff():
+ print(''.join(i))
+ else:
+ # No proper API function to perform the same operation.
+ print(subprocess.check_output(
+ ' '.join(['cd', path, '&&', 'osc', 'addremove']), shell=True))
+ package.commit(msg='Automatic update')
+
if __name__ == "__main__":
app = CommandLineInterface()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/pkglistgen.sh
new/openSUSE-release-tools-20180102.98a659c/pkglistgen.sh
--- old/openSUSE-release-tools-20171226.96a3eec/pkglistgen.sh 2017-12-26
12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/pkglistgen.sh 1970-01-01
01:00:00.000000000 +0100
@@ -1,146 +0,0 @@
-#!/bin/bash
-# FIXME: 000package-groups is not frozen, osc up doesn't do anything
-# when updated in underlying project
-
-set -e
-shopt -s nullglob
-
-self=$(readlink -e $(type -p "$0"))
-
-: ${project:=openSUSE:Factory}
-: ${api:=api.opensuse.org}
-: ${repos:=$project/standard}
-: ${productrepo:=standard}
-: ${arch:=x86_64}
-
-groups="000package-groups"
-product="000product"
-releases="000release-packages"
-
-cachedir=${XDG_CACHE_HOME:-~/.cache}/opensuse-packagelists/$api/$project
-todo=("$product" "$groups")
-solveargs=()
-
-if [ -n "$IGNORE_RECOMMENDED" ]; then
- solveargs+=('--ignore-recommended')
-fi
-if [ -n "$INCLUDE_SUGGESTED" ]; then
- solveargs+=('--include-suggested')
-fi
-if [ -n "$LOCALES_FROM" ]; then
- solveargs+=('--locales-from', "$LOCALES_FROM")
-fi
-
-_osc=`type -p osc`
-osc()
-{
- "$_osc" -A "https://$api"; "$@"
-}
-
-checkin() {
- if [ -n "$dryrun" ]; then
- osc diff
- else
- osc addremove
- osc ci -m "Automatic update"
- fi
-}
-
-if ! osc api "/source/$project/" | grep -q "$product" ; then
- osc undelete -m revive "$project/$product"
- # FIXME: build disable it
- echo "$product undeleted, skip dvd until next cycle"
- exit 0
-elif [ -z "$FORCE" ]; then
- bs_status=`osc api
"/build/$project/_result?package=$product&repository=$productrepo"`
- if echo "${bs_status}" | grep -q 'building\|dirty'; then
- echo "$project build in progress, skipping."
- exit 0
- fi
-fi
-
-mkdir -p "$cachedir"
-cd "$cachedir"
-
-if [ -z "$skip_releases" ]; then
- todo+=("$releases")
- if ! osc api "/source/$project/" | grep -q "$releases" ; then
- osc undelete -m revive "$project/$releases"
- echo "$releases undeleted, skip dvd until next cycle"
- exit 0
- fi
-fi
-# update package checkouts
-for i in "${todo[@]}"; do
- if [ ! -e "$i" ]; then
- osc co -c "$project/$i"
- fi
- pushd "$i"
- if ! osc status; then
- # merge conflict etc, try to check out new
- popd
- rm -rf "$i"
- osc co -c "$project/$i"
- else
- osc up
- popd
- fi
-done
-
-[ -z "$releases" ] || rm -f "$cachedir/$releases"/*
-cd "$cachedir/$product"
-rm -f -- *
-cp .osc/_service .
-cp "$cachedir/$groups"/* .
-rm -f supportstatus.txt groups.yml package-groups.changes
-for i in *.spec.in; do
- mv -v $i "${i%.in}"
-done
-if ! ${self%.sh}.py -i "$cachedir/$groups" -r $repos -o . -a $arch update; then
- echo "no change in packages"
-fi
-${self%.sh}.py -i "$cachedir/$groups" -r $repos -o . -a $arch solve
"${solveargs[@]}"
-for i in $delete_products; do
- rm -vf -- "$i"
-done
-for i in *.product; do
- /usr/lib/obs/service/create_single_product $PWD/$i $PWD $(cat .osc/_project)
-done
-for i in $delete_kiwis; do
- rm -vf -- "$i"
-done
-if [ -z "$skip_releases" ]; then
- mv -v *.spec "$cachedir/$releases"
-else
- rm -vf *.spec
-fi
-echo '<multibuild>' > _multibuild
-for file in *.kiwi; do
- container="${file##*/}"
- container="${container%.kiwi}"
- echo " <package>${container}</package>" >> _multibuild
-done
-echo '</multibuild>' >> _multibuild
-cat << EOF > stub.kiwi
-# prevent building single kiwi files twice
-Name: stub
-Version: 0.0
-EOF
-checkin
-
-if [ -z "$skip_releases" ]; then
- cd "$cachedir/$releases"
- echo '<multibuild>' > _multibuild
- for file in *.spec; do
- container="${file##*/}"
- container="${container%.spec}"
- echo " <package>${container}</package>" >> _multibuild
- done
- echo '</multibuild>' >> _multibuild
- cat <<-EOF > stub.spec
- # prevent building single spec files twice
- Name: stub
- Version: 0.0
- EOF
- checkin
-fi
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/repo_checker.py
new/openSUSE-release-tools-20180102.98a659c/repo_checker.py
--- old/openSUSE-release-tools-20171226.96a3eec/repo_checker.py 2017-12-26
12:36:26.000000000 +0100
+++ new/openSUSE-release-tools-20180102.98a659c/repo_checker.py 2018-01-02
11:39:25.000000000 +0100
@@ -42,13 +42,16 @@
self.force = False
self.limit_group = None
+ def repository_published(self, project):
+ root = ET.fromstringlist(show_results_meta(
+ self.apiurl, project, multibuild=True, repository=['standard']))
+ return not len(root.xpath('result[@state!="published"]'))
+
def project_only(self, project, post_comments=False):
# self.staging_config needed by target_archs().
api = self.staging_api(project)
- root = ET.fromstringlist(show_results_meta(
- self.apiurl, project, multibuild=True, repository=['standard']))
- if not self.force and len(root.xpath('result[@state!="published"]')):
+ if not self.force and self.repository_published(project):
self.logger.info('{}/standard not published'.format(project))
return
@@ -132,7 +135,7 @@
if not status or str(status['overall_state']) not in ('testing',
'review', 'acceptable'):
# Not in a "ready" state.
openQA_only = False # Not relevant so set to False.
- if str(status['overall_state']) == 'failed':
+ if status and str(status['overall_state']) == 'failed':
# Exception to the rule is openQA only in failed state.
openQA_only = True
for project in api.project_status_walk(status):
@@ -241,9 +244,12 @@
info_extra = {'build': self.groups_build[group]}
if not self.group_pass:
# Some checks in group did not pass, post comment.
+ # Avoid identical comments with different build hash during target
+ # project build phase. Once published update regardless.
+ published = self.repository_published(project)
self.comment_write(state='seen', result='failed', project=group,
message='\n'.join(comment).strip(),
identical=True,
- info_extra=info_extra)
+ info_extra=info_extra,
info_extra_identical=published)
else:
# Post passed comment only if previous failed comment.
text = 'Previously reported problems have been resolved.'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0
new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0
---
old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0
2017-12-26 12:36:26.000000000 +0100
+++
new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0
1970-01-01 01:00:00.000000000 +0100
@@ -1,3 +0,0 @@
-#!/bin/bash
-export project=openSUSE:Leap:15.0
-osrt-pkglistgen
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0-all
new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0-all
---
old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0-all
2017-12-26 12:36:26.000000000 +0100
+++
new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0-all
1970-01-01 01:00:00.000000000 +0100
@@ -1,6 +0,0 @@
-#!/bin/bash
-export LOCALES_FROM=openSUSE-product
-export INCLUDE_SUGGESTED=1
-osrt-pkglistgen-openSUSE:Leap:15.0
-osrt-pkglistgen-openSUSE:Leap:15.0:Rings
-osrt-pkglistgen-openSUSE:Leap:15.0:Staging
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh
new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh
---
old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh
2017-12-26 12:36:26.000000000 +0100
+++
new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Ports-aarch64.sh
1970-01-01 01:00:00.000000000 +0100
@@ -1,7 +0,0 @@
-#!/bin/bash
-self=$(readlink $(type -p "$0"))
-export project=openSUSE:Leap:15.0:Ports
-export repos=$project/ports
-export arch=aarch64
-export productrepo=ports
-osrt-pkglistgen
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Rings
new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Rings
---
old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Rings
2017-12-26 12:36:26.000000000 +0100
+++
new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Rings
1970-01-01 01:00:00.000000000 +0100
@@ -1,12 +0,0 @@
-#!/bin/bash
-main=openSUSE:Leap:15.0
-export delete_kiwis="openSUSE-ftp-ftp-x86_64.kiwi openSUSE-cd-mini-x86_64.kiwi"
-
-export project=$main:Rings:1-MinimalX
-export repos=$project/standard,$main:Rings:0-Bootstrap/standard
-osrt-pkglistgen
-
-export project=$main:Rings:2-TestDVD
-export repos=$main:Rings:2-TestDVD/standard,$repos
-export skip_releases=1
-osrt-pkglistgen
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Staging
new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Staging
---
old/openSUSE-release-tools-20171226.96a3eec/script/osrt-pkglistgen-openSUSE:Leap:15.0:Staging
2017-12-26 12:36:26.000000000 +0100
+++
new/openSUSE-release-tools-20180102.98a659c/script/osrt-pkglistgen-openSUSE:Leap:15.0:Staging
1970-01-01 01:00:00.000000000 +0100
@@ -1,20 +0,0 @@
-#!/bin/bash
-self=$(readlink $(type -p "$0"))
-main=openSUSE:Leap:15.0
-: ${letter:=A B C D E}
-
-export delete_kiwis="openSUSE-ftp-ftp-x86_64.kiwi openSUSE-cd-mini-x86_64.kiwi"
-for l in $letter; do
- export project=$main:Staging:$l
- echo "checking $project..."
- export repos=$project/standard
- if [ "$l" != A -a "$l" != B ]; then
- repos="$repos,$project/bootstrap_copy"
- fi
-
- # DVD project first as it depends on the project below, so might look
- # busy if we update the other one first
- project=$project:DVD repos=$project/standard,$repos skip_releases=1
osrt-pkglistgen
-
- osrt-pkglistgen
-done
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20171226.96a3eec/systemd/osrt-pkglistgen@.service
new/openSUSE-release-tools-20180102.98a659c/systemd/osrt-pkglistgen@.service
---
old/openSUSE-release-tools-20171226.96a3eec/systemd/osrt-pkglistgen@.service
2017-12-26 12:36:26.000000000 +0100
+++
new/openSUSE-release-tools-20180102.98a659c/systemd/osrt-pkglistgen@.service
2018-01-02 11:39:25.000000000 +0100
@@ -3,7 +3,7 @@
[Service]
User=osrt-repo-checker
-ExecStart=/usr/bin/osrt-pkglistgen-%i-all
+ExecStart=/usr/bin/osrt-pkglistgen -p "%i"
[Install]
WantedBy=multi-user.target
++++++ openSUSE-release-tools.obsinfo ++++++
--- /var/tmp/diff_new_pack.qsXQKW/_old 2018-01-02 16:35:52.334776989 +0100
+++ /var/tmp/diff_new_pack.qsXQKW/_new 2018-01-02 16:35:52.338775656 +0100
@@ -1,5 +1,5 @@
name: openSUSE-release-tools
-version: 20171226.96a3eec
-mtime: 1514288186
-commit: 96a3eec425d49f995a85e000da4a8799b5a6a461
+version: 20180102.98a659c
+mtime: 1514889565
+commit: 98a659c3fab8dc036f9945854eae93265d695bed
