Hello community,
here is the log from the commit of package openSUSE-release-tools for
openSUSE:Factory checked in at 2019-03-20 13:21:52
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/openSUSE-release-tools (Old)
and /work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.28833 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openSUSE-release-tools"
Wed Mar 20 13:21:52 2019 rev:169 rq:686859 version:20190320.d9b4fee
Changes:
--------
---
/work/SRC/openSUSE:Factory/openSUSE-release-tools/openSUSE-release-tools.changes
2019-03-14 15:02:30.655676462 +0100
+++
/work/SRC/openSUSE:Factory/.openSUSE-release-tools.new.28833/openSUSE-release-tools.changes
2019-03-20 13:22:20.385299122 +0100
@@ -1,0 +2,29 @@
+Wed Mar 20 07:35:39 UTC 2019 - [email protected]
+
+- Update to version 20190320.d9b4fee:
+ * gocd: Fix PYTHONPATH for the monitor scripts
+
+-------------------------------------------------------------------
+Wed Mar 20 07:26:09 UTC 2019 - [email protected]
+
+- Update to version 20190320.c76e0d9:
+ * Fix compat with pyYaml 5.1
+ * origin-manager: include origin annotation in comment when waiting.
+
+-------------------------------------------------------------------
+Wed Mar 20 07:20:02 UTC 2019 - [email protected]
+
+- Update to version 20190320.ab8a614:
+ * No longer package rabbit-openqa but add gocd config for it
+ * Add rabbit-repoid
+ * PubsubConsumer: Implement max runtime
+
+-------------------------------------------------------------------
+Wed Mar 20 05:38:05 UTC 2019 - [email protected]
+
+- Update to version 20190320.bb48831:
+ * Replace deprecated (and not advised) yaml.load() with yaml.safe_load().
+ * [openqa-maintenance] Add openSUSE Leap 15.1 configuration
+ * osclib/origin: policy_input_evaluate(): do not list disallowed reviews.
+
+-------------------------------------------------------------------
Old:
----
openSUSE-release-tools-20190313.b7eeea5.obscpio
New:
----
openSUSE-release-tools-20190320.d9b4fee.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ openSUSE-release-tools.spec ++++++
--- /var/tmp/diff_new_pack.V7jffJ/_old 2019-03-20 13:22:21.473298854 +0100
+++ /var/tmp/diff_new_pack.V7jffJ/_new 2019-03-20 13:22:21.481298853 +0100
@@ -20,7 +20,7 @@
%define source_dir openSUSE-release-tools
%define announcer_filename factory-package-news
Name: openSUSE-release-tools
-Version: 20190313.b7eeea5
+Version: 20190320.d9b4fee
Release: 0
Summary: Tools to aid in staging and release work for openSUSE/SUSE
License: GPL-2.0-or-later AND MIT
@@ -326,18 +326,6 @@
%description -n osc-plugin-vdelreq
OSC plugin to check for virtually accepted request, see `osc vdelreq --help`.
-%package rabbit-openqa
-Summary: Sync openQA Status Into OBS
-Group: Development/Tools/Other
-BuildArch: noarch
-Requires: osc >= 0.159.0
-Requires: python2-openqa_client
-Requires: python2-pika
-
-%description rabbit-openqa
-Bot listening to AMQP bus and syncs openQA job status into OBS for
-staging projects
-
%prep
%setup -q
@@ -452,14 +440,6 @@
%postun pkglistgen
%systemd_postun
-%pre rabbit-openqa
-getent passwd osrt-rabbit-openqa > /dev/null || \
- useradd -r -m -s /sbin/nologin -c "user for
openSUSE-release-tools-rabbit-openqa" osrt-rabbit-openqa
-exit 0
-
-%postun rabbit-openqa
-%systemd_postun
-
%files
%defattr(-,root,root,-)
%doc README.md
@@ -511,7 +491,6 @@
%exclude %{_datadir}/%{source_dir}/osc-staging.py
%exclude %{_datadir}/%{source_dir}/osc-vdelreq.py
%exclude %{_datadir}/%{source_dir}/update_crawler.py
-%exclude %{_datadir}/%{source_dir}/rabbit-openqa.py
%dir %{_sysconfdir}/openSUSE-release-tools
%files devel
@@ -660,12 +639,6 @@
%{_unitdir}/[email protected]
%{_unitdir}/[email protected]
-%files rabbit-openqa
-%defattr(-,root,root,-)
-%{_bindir}/osrt-rabbit-openqa
-%{_datadir}/%{source_dir}/rabbit-openqa.py
-%{_unitdir}/osrt-rabbit-openqa.service
-
%files -n osclib
%defattr(-,root,root,-)
%{_datadir}/%{source_dir}/osclib
++++++ _servicedata ++++++
--- /var/tmp/diff_new_pack.V7jffJ/_old 2019-03-20 13:22:21.553298835 +0100
+++ /var/tmp/diff_new_pack.V7jffJ/_new 2019-03-20 13:22:21.553298835 +0100
@@ -1,6 +1,6 @@
<servicedata>
<service name="tar_scm">
<param
name="url">https://github.com/openSUSE/openSUSE-release-tools.git</param>
- <param
name="changesrevision">b7eeea5d9817ca60c2ce76a0bc5a2455eb054561</param>
+ <param
name="changesrevision">d9b4fee58f5aefef3ce77f9b5fc7333e8683ad86</param>
</service>
</servicedata>
++++++ openSUSE-release-tools-20190313.b7eeea5.obscpio ->
openSUSE-release-tools-20190320.d9b4fee.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/PubSubConsumer.py
new/openSUSE-release-tools-20190320.d9b4fee/PubSubConsumer.py
--- old/openSUSE-release-tools-20190313.b7eeea5/PubSubConsumer.py
2019-03-13 11:54:59.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/PubSubConsumer.py
2019-03-20 08:30:21.000000000 +0100
@@ -1,6 +1,7 @@
import logging
import pika
import sys
+import time
from datetime import datetime
class PubSubConsumer(object):
@@ -30,6 +31,7 @@
self._consumer_tag = None
self._prefix = amqp_prefix
self._timer_id = None
+ self._run_until = None
self.logger = logger
def restart_timer(self):
@@ -45,7 +47,10 @@
def still_alive(self):
# output something so gocd doesn't consider it stalled
self.logger.info('Still alive: {}'.format(datetime.now().time()))
- self.restart_timer()
+ if self._run_until and time.time() > self._run_until:
+ self.stop()
+ else:
+ self.restart_timer()
def connect(self):
"""This method connects to RabbitMQ, returning the connection handle.
@@ -315,11 +320,13 @@
self.logger.debug('Creating a new channel')
self._connection.channel(on_open_callback=self.on_channel_open)
- def run(self):
+ def run(self, runtime=None):
"""Run the example consumer by connecting to RabbitMQ and then
starting the IOLoop to block and allow the SelectConnection to operate.
"""
+ if runtime:
+ self._run_until = time.time() + runtime
self._connection = self.connect()
self._connection.ioloop.start()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/data/apimap.json
new/openSUSE-release-tools-20190320.d9b4fee/data/apimap.json
--- old/openSUSE-release-tools-20190313.b7eeea5/data/apimap.json
2019-03-13 11:54:59.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/data/apimap.json
2019-03-20 08:30:21.000000000 +0100
@@ -138,5 +138,10 @@
"version": "15.0",
"flavor": "DVD-Incidents",
"distri": "opensuse"
+ },
+ "openSUSE:Leap:15.1:Update" : {
+ "version": "15.1",
+ "flavor": "DVD-Incidents",
+ "distri": "opensuse"
}
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/data/incidents.json
new/openSUSE-release-tools-20190320.d9b4fee/data/incidents.json
--- old/openSUSE-release-tools-20190313.b7eeea5/data/incidents.json
2019-03-13 11:54:59.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/data/incidents.json
2019-03-20 08:30:21.000000000 +0100
@@ -151,5 +151,11 @@
"FLAVOR": "DVD-Incidents",
"VERSION": "15.0",
"ARCH": "x86_64"
+ },
+ "openSUSE:Leap:15.1:Update": {
+ "DISTRI": "opensuse",
+ "FLAVOR": "DVD-Incidents",
+ "VERSION": "15.1",
+ "ARCH": "x86_64"
}
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/data/repos.json
new/openSUSE-release-tools-20190320.d9b4fee/data/repos.json
--- old/openSUSE-release-tools-20190313.b7eeea5/data/repos.json 2019-03-13
11:54:59.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/data/repos.json 2019-03-20
08:30:21.000000000 +0100
@@ -27,6 +27,20 @@
"http://download.opensuse.org/update/leap/15.0/oss/";,
"http://download.opensuse.org/update/leap/15.0/non-oss/";
]
+ },
+ "openSUSE:Leap:15.1:Update" : {
+ "settings" : {
+ "OS_TEST_ISSUES" : "",
+ "FLAVOR" : "DVD-Updates",
+ "DISTRI" : "opensuse",
+ "VERSION" : "15.1",
+ "ARCH" : "x86_64"
+ },
+ "test" : "kde",
+ "repos" : [
+ "http://download.opensuse.org/update/leap/15.1/oss/";,
+ "http://download.opensuse.org/update/leap/15.1/non-oss/";
+ ]
}
},
"https://openqa.suse.de"; : {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/dist/package/openSUSE-release-tools.spec
new/openSUSE-release-tools-20190320.d9b4fee/dist/package/openSUSE-release-tools.spec
---
old/openSUSE-release-tools-20190313.b7eeea5/dist/package/openSUSE-release-tools.spec
2019-03-13 11:54:59.000000000 +0100
+++
new/openSUSE-release-tools-20190320.d9b4fee/dist/package/openSUSE-release-tools.spec
2019-03-20 08:30:21.000000000 +0100
@@ -326,18 +326,6 @@
%description -n osc-plugin-vdelreq
OSC plugin to check for virtually accepted request, see `osc vdelreq --help`.
-%package rabbit-openqa
-Summary: Sync openQA Status Into OBS
-Group: Development/Tools/Other
-BuildArch: noarch
-Requires: osc >= 0.159.0
-Requires: python2-openqa_client
-Requires: python2-pika
-
-%description rabbit-openqa
-Bot listening to AMQP bus and syncs openQA job status into OBS for
-staging projects
-
%prep
%setup -q
@@ -452,14 +440,6 @@
%postun pkglistgen
%systemd_postun
-%pre rabbit-openqa
-getent passwd osrt-rabbit-openqa > /dev/null || \
- useradd -r -m -s /sbin/nologin -c "user for
openSUSE-release-tools-rabbit-openqa" osrt-rabbit-openqa
-exit 0
-
-%postun rabbit-openqa
-%systemd_postun
-
%files
%defattr(-,root,root,-)
%doc README.md
@@ -511,7 +491,6 @@
%exclude %{_datadir}/%{source_dir}/osc-staging.py
%exclude %{_datadir}/%{source_dir}/osc-vdelreq.py
%exclude %{_datadir}/%{source_dir}/update_crawler.py
-%exclude %{_datadir}/%{source_dir}/rabbit-openqa.py
%dir %{_sysconfdir}/openSUSE-release-tools
%files devel
@@ -660,12 +639,6 @@
%{_unitdir}/[email protected]
%{_unitdir}/[email protected]
-%files rabbit-openqa
-%defattr(-,root,root,-)
-%{_bindir}/osrt-rabbit-openqa
-%{_datadir}/%{source_dir}/rabbit-openqa.py
-%{_unitdir}/osrt-rabbit-openqa.service
-
%files -n osclib
%defattr(-,root,root,-)
%{_datadir}/%{source_dir}/osclib
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/gocd/monitors.gocd.yaml
new/openSUSE-release-tools-20190320.d9b4fee/gocd/monitors.gocd.yaml
--- old/openSUSE-release-tools-20190313.b7eeea5/gocd/monitors.gocd.yaml
1970-01-01 01:00:00.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/gocd/monitors.gocd.yaml
2019-03-20 08:30:21.000000000 +0100
@@ -0,0 +1,121 @@
+format_version: 3
+pipelines:
+ openSUSE.openQA:
+ group: Monitors
+ lock_behavior: unlockWhenFinished
+ timer:
+ spec: 0 */5 * ? * *
+ environment_variables:
+ OSC_CONFIG: /home/go/config/oscrc-staging-bot
+ materials:
+ git:
+ git: https://github.com/openSUSE/openSUSE-release-tools.git
+ branch: master
+ destination: scripts
+ stages:
+ - Run:
+ approval:
+ type: manual
+ jobs:
+ Run:
+ timeout: 0
+ resources:
+ - staging-bot
+ tasks:
+ - script: |-
+ export PYTHONPATH=$PWD/scripts
+ ./scripts/gocd/rabbit-openqa.py -A https://api.opensuse.org
+ SUSE.openQA:
+ group: Monitors
+ lock_behavior: unlockWhenFinished
+ timer:
+ spec: 0 */5 * ? * *
+ environment_variables:
+ OSC_CONFIG: /home/go/config/oscrc-staging-bot
+ materials:
+ git:
+ git: https://github.com/openSUSE/openSUSE-release-tools.git
+ branch: master
+ destination: scripts
+ stages:
+ - Run:
+ approval:
+ type: manual
+ jobs:
+ Run:
+ timeout: 0
+ resources:
+ - staging-bot
+ tasks:
+ # endless loop
+ - script: |-
+ export PYTHONPATH=$PWD/scripts
+ ./scripts/gocd/rabbit-openqa.py -A https://api.suse.de
+ SUSE.Repo.Monitor:
+ group: Monitors
+ lock_behavior: unlockWhenFinished
+ timer:
+ spec: 0 */5 * ? * *
+ environment_variables:
+ OSC_CONFIG: /home/go/config/oscrc-staging-bot
+ materials:
+ scripts:
+ git: https://github.com/openSUSE/openSUSE-release-tools.git
+ branch: master
+ destination: scripts
+ osc:
+ git: https://github.com/openSUSE/osc.git
+ branch: python3
+ destination: osc
+ stages:
+ - Run:
+ approval:
+ type: manual
+ jobs:
+ Run:
+ timeout: 0
+ resources:
+ - staging-bot
+ tasks:
+ # endless loop
+ - script: |-
+ export PYTHONPATH=$PWD/osc:$PWD/scripts
+ git config --global user.email "[email protected]"
+ git config --global user.name "GoCD Repo Monitor"
+ git clone git://botmaster.suse.de/suse-repos.git
+ cd suse-repos
+ python3 ../scripts/gocd/rabbit-repoid.py -A
https://api.suse.de SUSE:SLE-15-SP1:GA
+ openSUSE.Repo.Monitor:
+ group: Monitors
+ lock_behavior: unlockWhenFinished
+ timer:
+ spec: 0 */5 * ? * *
+ environment_variables:
+ OSC_CONFIG: /home/go/config/oscrc-staging-bot
+ materials:
+ scripts:
+ git: https://github.com/openSUSE/openSUSE-release-tools.git
+ branch: master
+ destination: scripts
+ osc:
+ git: https://github.com/openSUSE/osc.git
+ branch: python3
+ destination: osc
+ stages:
+ - Run:
+ approval:
+ type: manual
+ jobs:
+ Run:
+ timeout: 0
+ resources:
+ - staging-bot
+ tasks:
+ # endless loop
+ - script: |-
+ export PYTHONPATH=$PWD/osc:$PWD/scripts
+ git config --global user.email "[email protected]"
+ git config --global user.name "GoCD Repo Monitor"
+ git clone git://botmaster.suse.de/opensuse-repos.git
+ cd opensuse-repos
+ python3 ../scripts/gocd/rabbit-repoid.py -A
https://api.opensuse.org openSUSE:Factory openSUSE:Leap
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/gocd/rabbit-openqa.py
new/openSUSE-release-tools-20190320.d9b4fee/gocd/rabbit-openqa.py
--- old/openSUSE-release-tools-20190313.b7eeea5/gocd/rabbit-openqa.py
1970-01-01 01:00:00.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/gocd/rabbit-openqa.py
2019-03-20 08:30:21.000000000 +0100
@@ -0,0 +1,269 @@
+#!/usr/bin/python
+
+import argparse
+import logging
+import pika
+import sys
+import json
+import osc
+import re
+from time import sleep
+from osc.core import http_GET, http_POST, makeurl
+from M2Crypto.SSL import SSLError as SSLError
+from osclib.conf import Config
+from osclib.stagingapi import StagingAPI
+from lxml import etree as ET
+from openqa_client.client import OpenQA_Client
+from openqa_client.exceptions import ConnectionError
+try:
+ from urllib.error import HTTPError, URLError
+ from urllib.parse import quote_plus
+except ImportError:
+ # python 2.x
+ from urllib2 import HTTPError, URLError
+ from urllib import quote_plus
+
+import requests
+from PubSubConsumer import PubSubConsumer
+
+
+class Project(object):
+ def __init__(self, name):
+ self.name = name
+ Config(apiurl, name)
+ self.api = StagingAPI(apiurl, name)
+ self.staging_projects = dict()
+ self.listener = None
+ self.logger = logging.getLogger(__name__)
+ self.replace_string = self.api.attribute_value_load('OpenQAMapping')
+
+ def init(self):
+ for p in self.api.get_staging_projects():
+ if self.api.is_adi_project(p):
+ continue
+ self.staging_projects[p] = self.initial_staging_state(p)
+ self.update_staging_status(p)
+
+ def staging_letter(self, name):
+ return name.split(':')[-1]
+
+ def map_iso(self, staging_project, iso):
+ parts = self.replace_string.split('/')
+ if parts[0] != 's':
+ raise Exception("{}'s iso_replace_string does not start with
s/".format(self.name))
+ old = parts[1]
+ new = parts[2]
+ new = new.replace('$LETTER', self.staging_letter(staging_project))
+ return re.compile(old).sub(new, iso)
+
+ def gather_isos(self, name, repository):
+ url = self.api.makeurl(['published', name, repository, 'iso'])
+ f = self.api.retried_GET(url)
+ root = ET.parse(f).getroot()
+ ret = []
+ for entry in root.findall('entry'):
+ if entry.get('name').endswith('iso'):
+ ret.append(self.map_iso(name, entry.get('name')))
+ return ret
+
+ def gather_buildid(self, name, repository):
+ url = self.api.makeurl(['published', name, repository], {'view':
'status'})
+ f = self.api.retried_GET(url)
+ id = ET.parse(f).getroot().find('buildid')
+ if id is not None:
+ return id.text
+
+ def initial_staging_state(self, name):
+ return {'isos': self.gather_isos(name, 'images'),
+ 'id': self.gather_buildid(name, 'images')}
+
+ def fetch_openqa_jobs(self, staging, iso):
+ buildid = self.staging_projects[staging].get('id')
+ if not buildid:
+ self.logger.info("I don't know the build id of " + staging)
+ return
+ # all openQA jobs are created at the same URL
+ url = self.api.makeurl(['status_reports', 'published', staging,
'images', 'reports', buildid])
+ openqa = self.listener.jobs_for_iso(iso)
+ # collect job infos to pick names
+ openqa_infos = dict()
+ for job in openqa:
+ print(staging, iso, job['id'], job['state'], job['result'],
+ job['settings']['MACHINE'], job['settings']['TEST'])
+ openqa_infos[job['id']] = {'url': self.listener.test_url(job)}
+ openqa_infos[job['id']]['state'] = self.map_openqa_result(job)
+ openqa_infos[job['id']]['name'] = job['settings']['TEST']
+ openqa_infos[job['id']]['machine'] = job['settings']['MACHINE']
+
+ # make sure the names are unique
+ taken_names = dict()
+ for id in openqa_infos:
+ name = openqa_infos[id]['name']
+ if name in taken_names:
+ openqa_infos[id]['name'] = openqa_infos[id]['name'] + "@" +
openqa_infos[id]['machine']
+ # the other id
+ id = taken_names[name]
+ openqa_infos[id]['name'] = openqa_infos[id]['name'] + "@" +
openqa_infos[id]['machine']
+ taken_names[name] = id
+
+ for info in openqa_infos.values():
+ xml = self.openqa_check_xml(info['url'], info['state'], 'openqa:'
+ info['name'])
+ try:
+ http_POST(url, data=xml)
+ except HTTPError:
+ self.logger.error('failed to post status to ' + url)
+
+ def update_staging_status(self, project):
+ for iso in self.staging_projects[project]['isos']:
+ self.fetch_openqa_jobs(project, iso)
+
+ def update_staging_buildid(self, project, repository, buildid):
+ self.staging_projects[project]['id'] = buildid
+ self.staging_projects[project]['isos'] = self.gather_isos(project,
repository)
+ self.update_staging_status(project)
+
+ def check_published_repo(self, project, repository, buildid):
+ if repository != 'images':
+ return
+ for p in self.staging_projects:
+ if project == p:
+ self.update_staging_buildid(project, repository, buildid)
+
+ def matching_project(self, iso):
+ for p in self.staging_projects:
+ if iso in self.staging_projects[p]['isos']:
+ return p
+
+ def map_openqa_result(self, job):
+ if job['result'] in ['passed', 'softfailed']:
+ return 'success'
+ if job['result'] == 'none':
+ return 'pending'
+ return 'failure'
+
+ def openqa_job_change(self, iso):
+ staging = self.matching_project(iso)
+ if not staging:
+ return
+ # we fetch all openqa jobs so we can avoid long job names
+ self.fetch_openqa_jobs(staging, iso)
+
+ def openqa_check_xml(self, url, state, name):
+ check = ET.Element('check')
+ se = ET.SubElement(check, 'url')
+ se.text = url
+ se = ET.SubElement(check, 'state')
+ se.text = state
+ se = ET.SubElement(check, 'name')
+ se.text = name
+ return ET.tostring(check)
+
+
+class Listener(PubSubConsumer):
+ def __init__(self, amqp_prefix, openqa_url):
+ super(Listener, self).__init__(amqp_prefix,
logging.getLogger(__name__))
+ self.projects = []
+ self.amqp_prefix = amqp_prefix
+ self.openqa_url = openqa_url
+ self.openqa = OpenQA_Client(server=openqa_url)
+
+ def routing_keys(self):
+ ret = []
+ for suffix in ['.obs.repo.published', '.openqa.job.done',
+ '.openqa.job.create', '.openqa.job.restart']:
+ ret.append(self.amqp_prefix + suffix)
+ return ret
+
+ def add(self, project):
+ project.listener = self
+ self.projects.append(project)
+
+ def start_consuming(self):
+ # now we are (re-)connected to the bus and need to fetch the
+ # initial state
+ for project in self.projects:
+ self.logger.info('Fetching ISOs of %s', project.name)
+ project.init()
+ self.logger.info('Finished fetching initial ISOs, listening')
+ super(Listener, self).start_consuming()
+
+ def jobs_for_iso(self, iso):
+ values = {
+ 'iso': iso,
+ 'scope': 'current',
+ 'latest': '1',
+ }
+ return self.openqa.openqa_request('GET', 'jobs', values)['jobs']
+
+ def get_step_url(self, testurl, modulename):
+ failurl = testurl +
'/modules/{!s}/fails'.format(quote_plus(modulename))
+ fails = requests.get(failurl).json()
+ failed_step = fails.get('first_failed_step', 1)
+ return "{!s}#step/{!s}/{:d}".format(testurl, modulename, failed_step)
+
+ def test_url(self, job):
+ url = self.openqa_url + ("/tests/%d" % job['id'])
+ if job['result'] == 'failed':
+ for module in job['modules']:
+ if module['result'] == 'failed':
+ return self.get_step_url(url, module['name'])
+ return url
+
+ def on_published_repo(self, payload):
+ for p in self.projects:
+ p.check_published_repo(str(payload['project']),
str(payload['repo']), str(payload['buildid']))
+
+ def on_openqa_job(self, iso):
+ self.logger.debug('openqa_job_change %s', iso)
+ for p in self.projects:
+ p.openqa_job_change(iso)
+
+ def on_message(self, unused_channel, method, properties, body):
+ if method.routing_key == '{}.obs.repo.published'.format(amqp_prefix):
+ self.on_published_repo(json.loads(body))
+ elif re.search(r'.openqa.', method.routing_key):
+ self.on_openqa_job(json.loads(body).get('ISO'))
+ else:
+ self.logger.warning("unknown rabbitmq message
{}".format(method.routing_key))
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(
+ description='Bot to sync openQA status to OBS')
+ parser.add_argument("--apiurl", '-A', type=str, help='API URL of OBS')
+ parser.add_argument('-s', '--staging', type=str, default=None,
+ help='staging project letter')
+ parser.add_argument('-f', '--force', action='store_true', default=False,
+ help='force the write of the comment')
+ parser.add_argument('-p', '--project', type=str, default='Factory',
+ help='openSUSE version to make the check (Factory,
13.2)')
+ parser.add_argument('-d', '--debug', action='store_true', default=False,
+ help='enable debug information')
+
+ args = parser.parse_args()
+
+ osc.conf.get_config(override_apiurl = args.apiurl)
+ osc.conf.config['debug'] = args.debug
+
+ apiurl = osc.conf.config['apiurl']
+
+ if apiurl.endswith('suse.de'):
+ amqp_prefix = 'suse'
+ openqa_url = 'https://openqa.suse.de'
+ else:
+ amqp_prefix = 'opensuse'
+ openqa_url = 'https://openqa.opensuse.org'
+
+ logging.basicConfig(level=logging.INFO)
+
+ l = Listener(amqp_prefix, openqa_url)
+ url = makeurl(apiurl, ['search', 'project', 'id'], {'match':
'attribute/@name="OSRT:OpenQAMapping"'})
+ f = http_GET(url)
+ root = ET.parse(f).getroot()
+ for entry in root.findall('project'):
+ l.add(Project(entry.get('name')))
+
+ try:
+ l.run(runtime=3600)
+ except KeyboardInterrupt:
+ l.stop()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/gocd/rabbit-repoid.py
new/openSUSE-release-tools-20190320.d9b4fee/gocd/rabbit-repoid.py
--- old/openSUSE-release-tools-20190313.b7eeea5/gocd/rabbit-repoid.py
1970-01-01 01:00:00.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/gocd/rabbit-repoid.py
2019-03-20 08:30:21.000000000 +0100
@@ -0,0 +1,137 @@
+#! /usr/bin/python
+
+from __future__ import print_function
+
+import argparse
+import datetime
+import glob
+import json
+import logging
+import os.path
+import time
+
+import osc
+from osc.core import http_GET, makeurl
+
+from osclib.core import target_archs
+from lxml import etree as ET
+
+try:
+ from urllib.error import HTTPError
+except ImportError:
+ # python 2.x
+ from urllib2 import HTTPError
+
+from PubSubConsumer import PubSubConsumer
+
+class Listener(PubSubConsumer):
+ def __init__(self, apiurl, amqp_prefix, namespaces):
+ super(Listener, self).__init__(amqp_prefix,
logging.getLogger(__name__))
+ self.apiurl = apiurl
+ self.amqp_prefix = amqp_prefix
+ self.namespaces = namespaces
+
+ def routing_keys(self):
+ return [self.amqp_prefix + '.obs.repo.build_finished']
+
+ def check_arch(self, project, repository, architecture):
+ url = makeurl(self.apiurl, [
+ 'build', project, repository, architecture], {'view':
'status'})
+ root = ET.parse(http_GET(url)).getroot()
+ if root.get('code') == 'finished':
+ buildid = root.find('buildid')
+ if buildid is not None:
+ return buildid.text
+
+ def check_all_archs(self, project, repository):
+ ids = {}
+ try:
+ archs = target_archs(self.apiurl, project, repository)
+ except HTTPError:
+ return None
+ for arch in archs:
+ repoid = self.check_arch(project, repository, arch)
+ if not repoid:
+ self.logger.info('{}/{}/{} not yet done'.format(project,
repository, arch))
+ return None
+ ids[arch] = repoid
+ self.logger.info('All of {}/{} finished'.format(project, repository))
+ return ids
+
+ def is_part_of_namespaces(self, project):
+ for namespace in self.namespaces:
+ if project.startswith(namespace):
+ return True
+
+ def start_consuming(self):
+ # now we are (re-)connected to the bus and need to fetch the
+ # initial state
+ for namespace in self.namespaces:
+ for state in glob.glob('{}*.yaml'.format(namespace)):
+ state = state.replace('.yaml', '')
+ # split
+ project, repository = state.split('_-_')
+ self.update_repo(project, repository)
+ self.push_git('Restart of Repo Monitor')
+ self.logger.info('Finished refreshing repoids')
+ super(Listener, self).start_consuming()
+
+ def push_git(self, message):
+ os.system('git add . ')
+ os.system('git commit -m "{}" > /dev/null'.format(message))
+ os.system('git push > /dev/null')
+
+ def update_repo(self, project, repository):
+ ids = self.check_all_archs(project, repository)
+ if not ids:
+ return
+ pathname = project + '_-_' + repository + '.yaml'
+ with open(pathname, 'w') as f:
+ for arch in sorted(ids.keys()):
+ f.write('{}: {}\n'.format(arch, ids[arch]))
+
+ def on_message(self, unused_channel, method, properties, body):
+ try:
+ body = json.loads(body)
+ except ValueError:
+ return
+ if method.routing_key.endswith('.obs.repo.build_finished'):
+ if not self.is_part_of_namespaces(body['project']):
+ return
+ self.restart_timer()
+ self.logger.info('Repo finished event:
{}/{}/{}'.format(body['project'], body['repo'], body['arch']))
+ self.update_repo(body['project'], body['repo'])
+ self.push_git('Repository finished: {}/{}'.format(body['project'],
body['repo']))
+ else:
+ self.logger.warning(
+ 'unknown rabbitmq message {}'.format(method.routing_key))
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(
+ description='Monitor to commit repo status to git (for gocd trigger)')
+ parser.add_argument('--apiurl', '-A', type=str, help='API URL of OBS')
+ parser.add_argument('-d', '--debug', action='store_true', default=False,
+ help='enable debug information')
+ parser.add_argument('namespaces', nargs='*', help='namespaces to wait for')
+
+ args = parser.parse_args()
+
+ osc.conf.get_config(override_apiurl=args.apiurl)
+ osc.conf.config['debug'] = args.debug
+
+ apiurl = osc.conf.config['apiurl']
+
+ if apiurl.endswith('suse.de'):
+ amqp_prefix = 'suse'
+ else:
+ amqp_prefix = 'opensuse'
+
+ logging.basicConfig(level=logging.INFO)
+
+ listener = Listener(apiurl, amqp_prefix, args.namespaces)
+
+ try:
+ listener.run(3600)
+ except KeyboardInterrupt:
+ listener.stop()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/origin-manager.py
new/openSUSE-release-tools-20190320.d9b4fee/origin-manager.py
--- old/openSUSE-release-tools-20190313.b7eeea5/origin-manager.py
2019-03-13 11:54:59.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/origin-manager.py
2019-03-20 08:30:21.000000000 +0100
@@ -51,6 +51,11 @@
if len(result.reviews):
self.policy_result_reviews_add(project, package, result.reviews)
+ if result.wait:
+ # Since the review will not be accepted with the annotation
+ # containing origin context dump it the comment.
+ result.comments.insert(0, origin_annotation_dump(origin_info_new,
origin_info_old))
+
self.policy_result_comment_add(project, package, result.comments)
if result.wait:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/osclib/origin.py
new/openSUSE-release-tools-20190320.d9b4fee/osclib/origin.py
--- old/openSUSE-release-tools-20190313.b7eeea5/osclib/origin.py
2019-03-13 11:54:59.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/osclib/origin.py
2019-03-20 08:30:21.000000000 +0100
@@ -450,11 +450,9 @@
result = PolicyResult(wait, True, result.reviews, result.comments)
if wait:
- if policy['pending_submission_allow'] and len(reviews_not_allowed):
- result.comments.append('Waiting on reviews of {}:\n\n-
{}'.format(
- inputs['pending_submission'].identifier, '\n-
'.join(reviews_not_allowed)))
- else:
- result.comments.append('Waiting on
{}.'.format(inputs['pending_submission'].identifier))
+ result.comments.append('Waiting on {} of {}.'.format(
+ 'reviews' if policy['pending_submission_allow'] else
'acceptance',
+ inputs['pending_submission'].identifier))
if policy['maintainer_review_always']:
# Placed last to override initial maintainer approval message.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/osclib/stagingapi.py
new/openSUSE-release-tools-20190320.d9b4fee/osclib/stagingapi.py
--- old/openSUSE-release-tools-20190313.b7eeea5/osclib/stagingapi.py
2019-03-13 11:54:59.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/osclib/stagingapi.py
2019-03-20 08:30:21.000000000 +0100
@@ -698,7 +698,7 @@
def load_prj_pseudometa(self, description_text):
try:
- data = yaml.load(description_text)
+ data = yaml.safe_load(description_text)
if isinstance(data, str) or data is None:
data = {}
except (TypeError, AttributeError):
@@ -737,7 +737,7 @@
description = root.find('description')
# Order the requests and replace it with yaml
meta['requests'] = sorted(meta.get('requests', []), key=lambda x:
x['id'])
- yaml_new = yaml.dump(meta)
+ yaml_new = yaml.dump(meta, default_flow_style=None)
if yaml_new == description.text:
return
description.text = yaml_new
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/rabbit-openqa.py
new/openSUSE-release-tools-20190320.d9b4fee/rabbit-openqa.py
--- old/openSUSE-release-tools-20190313.b7eeea5/rabbit-openqa.py
2019-03-13 11:54:59.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/rabbit-openqa.py
1970-01-01 01:00:00.000000000 +0100
@@ -1,269 +0,0 @@
-#!/usr/bin/python
-
-import argparse
-import logging
-import pika
-import sys
-import json
-import osc
-import re
-from time import sleep
-from osc.core import http_GET, http_POST, makeurl
-from M2Crypto.SSL import SSLError as SSLError
-from osclib.conf import Config
-from osclib.stagingapi import StagingAPI
-from lxml import etree as ET
-from openqa_client.client import OpenQA_Client
-from openqa_client.exceptions import ConnectionError
-try:
- from urllib.error import HTTPError, URLError
- from urllib.parse import quote_plus
-except ImportError:
- # python 2.x
- from urllib2 import HTTPError, URLError
- from urllib import quote_plus
-
-import requests
-from PubSubConsumer import PubSubConsumer
-
-
-class Project(object):
- def __init__(self, name):
- self.name = name
- Config(apiurl, name)
- self.api = StagingAPI(apiurl, name)
- self.staging_projects = dict()
- self.listener = None
- self.logger = logging.getLogger(__name__)
- self.replace_string = self.api.attribute_value_load('OpenQAMapping')
-
- def init(self):
- for p in self.api.get_staging_projects():
- if self.api.is_adi_project(p):
- continue
- self.staging_projects[p] = self.initial_staging_state(p)
- self.update_staging_status(p)
-
- def staging_letter(self, name):
- return name.split(':')[-1]
-
- def map_iso(self, staging_project, iso):
- parts = self.replace_string.split('/')
- if parts[0] != 's':
- raise Exception("{}'s iso_replace_string does not start with
s/".format(self.name))
- old = parts[1]
- new = parts[2]
- new = new.replace('$LETTER', self.staging_letter(staging_project))
- return re.compile(old).sub(new, iso)
-
- def gather_isos(self, name, repository):
- url = self.api.makeurl(['published', name, repository, 'iso'])
- f = self.api.retried_GET(url)
- root = ET.parse(f).getroot()
- ret = []
- for entry in root.findall('entry'):
- if entry.get('name').endswith('iso'):
- ret.append(self.map_iso(name, entry.get('name')))
- return ret
-
- def gather_buildid(self, name, repository):
- url = self.api.makeurl(['published', name, repository], {'view':
'status'})
- f = self.api.retried_GET(url)
- id = ET.parse(f).getroot().find('buildid')
- if id is not None:
- return id.text
-
- def initial_staging_state(self, name):
- return {'isos': self.gather_isos(name, 'images'),
- 'id': self.gather_buildid(name, 'images')}
-
- def fetch_openqa_jobs(self, staging, iso):
- buildid = self.staging_projects[staging].get('id')
- if not buildid:
- self.logger.info("I don't know the build id of " + staging)
- return
- # all openQA jobs are created at the same URL
- url = self.api.makeurl(['status_reports', 'published', staging,
'images', 'reports', buildid])
- openqa = self.listener.jobs_for_iso(iso)
- # collect job infos to pick names
- openqa_infos = dict()
- for job in openqa:
- print(staging, iso, job['id'], job['state'], job['result'],
- job['settings']['MACHINE'], job['settings']['TEST'])
- openqa_infos[job['id']] = {'url': self.listener.test_url(job)}
- openqa_infos[job['id']]['state'] = self.map_openqa_result(job)
- openqa_infos[job['id']]['name'] = job['settings']['TEST']
- openqa_infos[job['id']]['machine'] = job['settings']['MACHINE']
-
- # make sure the names are unique
- taken_names = dict()
- for id in openqa_infos:
- name = openqa_infos[id]['name']
- if name in taken_names:
- openqa_infos[id]['name'] = openqa_infos[id]['name'] + "@" +
openqa_infos[id]['machine']
- # the other id
- id = taken_names[name]
- openqa_infos[id]['name'] = openqa_infos[id]['name'] + "@" +
openqa_infos[id]['machine']
- taken_names[name] = id
-
- for info in openqa_infos.values():
- xml = self.openqa_check_xml(info['url'], info['state'], 'openqa:'
+ info['name'])
- try:
- http_POST(url, data=xml)
- except HTTPError:
- self.logger.error('failed to post status to ' + url)
-
- def update_staging_status(self, project):
- for iso in self.staging_projects[project]['isos']:
- self.fetch_openqa_jobs(project, iso)
-
- def update_staging_buildid(self, project, repository, buildid):
- self.staging_projects[project]['id'] = buildid
- self.staging_projects[project]['isos'] = self.gather_isos(project,
repository)
- self.update_staging_status(project)
-
- def check_published_repo(self, project, repository, buildid):
- if repository != 'images':
- return
- for p in self.staging_projects:
- if project == p:
- self.update_staging_buildid(project, repository, buildid)
-
- def matching_project(self, iso):
- for p in self.staging_projects:
- if iso in self.staging_projects[p]['isos']:
- return p
-
- def map_openqa_result(self, job):
- if job['result'] in ['passed', 'softfailed']:
- return 'success'
- if job['result'] == 'none':
- return 'pending'
- return 'failure'
-
- def openqa_job_change(self, iso):
- staging = self.matching_project(iso)
- if not staging:
- return
- # we fetch all openqa jobs so we can avoid long job names
- self.fetch_openqa_jobs(staging, iso)
-
- def openqa_check_xml(self, url, state, name):
- check = ET.Element('check')
- se = ET.SubElement(check, 'url')
- se.text = url
- se = ET.SubElement(check, 'state')
- se.text = state
- se = ET.SubElement(check, 'name')
- se.text = name
- return ET.tostring(check)
-
-
-class Listener(PubSubConsumer):
- def __init__(self, amqp_prefix, openqa_url):
- super(Listener, self).__init__(amqp_prefix,
logging.getLogger(__name__))
- self.projects = []
- self.amqp_prefix = amqp_prefix
- self.openqa_url = openqa_url
- self.openqa = OpenQA_Client(server=openqa_url)
-
- def routing_keys(self):
- ret = []
- for suffix in ['.obs.repo.published', '.openqa.job.done',
- '.openqa.job.create', '.openqa.job.restart']:
- ret.append(self.amqp_prefix + suffix)
- return ret
-
- def add(self, project):
- project.listener = self
- self.projects.append(project)
-
- def start_consuming(self):
- # now we are (re-)connected to the bus and need to fetch the
- # initial state
- for project in self.projects:
- self.logger.info('Fetching ISOs of %s', project.name)
- project.init()
- self.logger.info('Finished fetching initial ISOs, listening')
- super(Listener, self).start_consuming()
-
- def jobs_for_iso(self, iso):
- values = {
- 'iso': iso,
- 'scope': 'current',
- 'latest': '1',
- }
- return self.openqa.openqa_request('GET', 'jobs', values)['jobs']
-
- def get_step_url(self, testurl, modulename):
- failurl = testurl +
'/modules/{!s}/fails'.format(quote_plus(modulename))
- fails = requests.get(failurl).json()
- failed_step = fails.get('first_failed_step', 1)
- return "{!s}#step/{!s}/{:d}".format(testurl, modulename, failed_step)
-
- def test_url(self, job):
- url = self.openqa_url + ("/tests/%d" % job['id'])
- if job['result'] == 'failed':
- for module in job['modules']:
- if module['result'] == 'failed':
- return self.get_step_url(url, module['name'])
- return url
-
- def on_published_repo(self, payload):
- for p in self.projects:
- p.check_published_repo(str(payload['project']),
str(payload['repo']), str(payload['buildid']))
-
- def on_openqa_job(self, iso):
- self.logger.debug('openqa_job_change %s', iso)
- for p in self.projects:
- p.openqa_job_change(iso)
-
- def on_message(self, unused_channel, method, properties, body):
- if method.routing_key == '{}.obs.repo.published'.format(amqp_prefix):
- self.on_published_repo(json.loads(body))
- elif re.search(r'.openqa.', method.routing_key):
- self.on_openqa_job(json.loads(body).get('ISO'))
- else:
- self.logger.warning("unknown rabbitmq message
{}".format(method.routing_key))
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser(
- description='Bot to sync openQA status to OBS')
- parser.add_argument("--apiurl", '-A', type=str, help='API URL of OBS')
- parser.add_argument('-s', '--staging', type=str, default=None,
- help='staging project letter')
- parser.add_argument('-f', '--force', action='store_true', default=False,
- help='force the write of the comment')
- parser.add_argument('-p', '--project', type=str, default='Factory',
- help='openSUSE version to make the check (Factory,
13.2)')
- parser.add_argument('-d', '--debug', action='store_true', default=False,
- help='enable debug information')
-
- args = parser.parse_args()
-
- osc.conf.get_config(override_apiurl = args.apiurl)
- osc.conf.config['debug'] = args.debug
-
- apiurl = osc.conf.config['apiurl']
-
- if apiurl.endswith('suse.de'):
- amqp_prefix = 'suse'
- openqa_url = 'https://openqa.suse.de'
- else:
- amqp_prefix = 'opensuse'
- openqa_url = 'https://openqa.opensuse.org'
-
- logging.basicConfig(level=logging.INFO)
-
- l = Listener(amqp_prefix, openqa_url)
- url = makeurl(apiurl, ['search', 'project', 'id'], {'match':
'attribute/@name="OSRT:OpenQAMapping"'})
- f = http_GET(url)
- root = ET.parse(f).getroot()
- for entry in root.findall('project'):
- l.add(Project(entry.get('name')))
-
- try:
- l.run()
- except KeyboardInterrupt:
- l.stop()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/systemd/osrt-rabbit-openqa.service
new/openSUSE-release-tools-20190320.d9b4fee/systemd/osrt-rabbit-openqa.service
---
old/openSUSE-release-tools-20190313.b7eeea5/systemd/osrt-rabbit-openqa.service
2019-03-13 11:54:59.000000000 +0100
+++
new/openSUSE-release-tools-20190320.d9b4fee/systemd/osrt-rabbit-openqa.service
1970-01-01 01:00:00.000000000 +0100
@@ -1,10 +0,0 @@
-[Unit]
-Description=openSUSE Release Tools: Sync openQA status
-
-[Service]
-User=osrt-rabbit-openqa
-ExecStart=/usr/bin/osrt-rabbit-openqa
-Restart=on-failure
-
-[Install]
-WantedBy=multi-user.target
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openSUSE-release-tools-20190313.b7eeea5/totest-manager.py
new/openSUSE-release-tools-20190320.d9b4fee/totest-manager.py
--- old/openSUSE-release-tools-20190313.b7eeea5/totest-manager.py
2019-03-13 11:54:59.000000000 +0100
+++ new/openSUSE-release-tools-20190320.d9b4fee/totest-manager.py
2019-03-20 08:30:21.000000000 +0100
@@ -94,7 +94,7 @@
def load_issues_to_ignore(self):
text = self.api.attribute_value_load('IgnoredIssues')
if text:
- root = yaml.load(text)
+ root = yaml.safe_load(text)
self.issues_to_ignore = root.get('last_seen')
else:
self.issues_to_ignore = dict()
++++++ openSUSE-release-tools.obsinfo ++++++
--- /var/tmp/diff_new_pack.V7jffJ/_old 2019-03-20 13:22:22.201298675 +0100
+++ /var/tmp/diff_new_pack.V7jffJ/_new 2019-03-20 13:22:22.201298675 +0100
@@ -1,5 +1,5 @@
name: openSUSE-release-tools
-version: 20190313.b7eeea5
-mtime: 1552474499
-commit: b7eeea5d9817ca60c2ce76a0bc5a2455eb054561
+version: 20190320.d9b4fee
+mtime: 1553067021
+commit: d9b4fee58f5aefef3ce77f9b5fc7333e8683ad86
