Hello community,
here is the log from the commit of package python-openqa_review for
openSUSE:Factory checked in at 2017-02-07 12:08:20
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-openqa_review (Old)
and /work/SRC/openSUSE:Factory/.python-openqa_review.new (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-openqa_review"
Changes:
--------
---
/work/SRC/openSUSE:Factory/python-openqa_review/python-openqa_review.changes
2017-02-03 18:57:56.900896765 +0100
+++
/work/SRC/openSUSE:Factory/.python-openqa_review.new/python-openqa_review.changes
2017-02-07 12:08:20.668222109 +0100
@@ -1,0 +2,18 @@
+Mon Feb 06 17:09:31 UTC 2017 - [email protected]
+
+- Update to version 1.5.0:
+ * Post issue reminder comments from daily review script
+ * tumblesle_release: Add AMQP notification support for 'suse_msg'
+ * tumblesle_release: Add documentation for AMQP notifications
+ * tumblesle_release: Prevent spammy repetition of AMQP notifications
+ * Get rid of duplicate update_jekyll
+ * More tumblesle notifications
+ * Fix tumblesle script not checking for new builds / updated job results
+ * Make tumblesle_release notification connection more resilient
+ * tumblesle_release: Only store in notify_seen on success
+ * Recommend safer installation proposals
+ * Ensure valid python certificates with certifi
+ * Be more specific about the error when file not found in cache
+ * Ensure all data is cached on save in openqa_review_osd_daily_email
+
+-------------------------------------------------------------------
Old:
----
python-openqa_review-1.4.1.tar.gz
New:
----
python-openqa_review-1.5.0.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-openqa_review.spec ++++++
--- /var/tmp/diff_new_pack.wSZm0W/_old 2017-02-07 12:08:21.188148544 +0100
+++ /var/tmp/diff_new_pack.wSZm0W/_new 2017-02-07 12:08:21.188148544 +0100
@@ -18,7 +18,7 @@
%define short_name openqa_review
Name: python-%{short_name}
-Version: 1.4.1
+Version: 1.5.0
Release: 0
Summary: A review helper script for openQA
License: MIT
++++++ python-openqa_review-1.4.1.tar.gz -> python-openqa_review-1.5.0.tar.gz
++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openqa_review-1.4.1/README.md
new/openqa_review-1.5.0/README.md
--- old/openqa_review-1.4.1/README.md 2017-01-20 14:23:49.000000000 +0100
+++ new/openqa_review-1.5.0/README.md 2017-02-06 18:06:38.000000000 +0100
@@ -6,11 +6,19 @@
## Usage
-* Install requirements and package, e.g.
+* Install requirements and package, using an isolated Python environment
+ such as
[VirtualEnv](http://docs.python-guide.org/en/latest/dev/virtualenvs/).
```
-sudo pip install -r requirements.txt
-sudo pip install .
+pip install -r requirements.txt
+pip install .
+```
+
+or if you are using openSUSE distribution, it is recommended to use `zypper`,
+e.g.:
+
+```
+zypper in python-openqa_review
```
* Call `openqa-review` from PATH, e.g. with `--help`
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openqa_review-1.4.1/bin/openqa_review_osd_daily_email
new/openqa_review-1.5.0/bin/openqa_review_osd_daily_email
--- old/openqa_review-1.4.1/bin/openqa_review_osd_daily_email 2017-01-20
14:23:49.000000000 +0100
+++ new/openqa_review-1.5.0/bin/openqa_review_osd_daily_email 2017-02-06
18:06:38.000000000 +0100
@@ -9,7 +9,8 @@
load_args="${load_args:-"--load --load-dir=${tmp}"}"
openqa_review_email_args="${openqa_review_email_args:-"${load_args}"}"
openqa_review_html_args="${openqa_review_html_args:-"${load_args}
--report-links"}"
-openqa_review_save_args="${openqa_review_save_args:-"--report-links --save
--save-dir ${tmp}"}"
+openqa_review_save_args="${openqa_review_save_args:-"--report-links
--reminder-comment-on-issues --dry-run --save --save-dir ${tmp}"}"
+openqa_review_reminder_args="${openqa_review_reminder_args:-"${load_args}
--reminder-comment-on-issues"}"
openqa_review="${openqa_review:-"$(which openqa-review)"}"
TPL="${TPL:-"dashboard_files/dashboard.html.in"}"
# the awk call is to ignore the "InsecureWarning" and the following blank line
@@ -48,3 +49,5 @@
$(<$TPL)
EOF
" > ${html_target}
+
+${openqa_review} $openqa_review_args $openqa_review_reminder_args > /dev/null
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openqa_review-1.4.1/bin/tumblesle-release-server-12sp3-x86_64
new/openqa_review-1.5.0/bin/tumblesle-release-server-12sp3-x86_64
--- old/openqa_review-1.4.1/bin/tumblesle-release-server-12sp3-x86_64
2017-01-20 14:23:49.000000000 +0100
+++ new/openqa_review-1.5.0/bin/tumblesle-release-server-12sp3-x86_64
2017-02-06 18:06:38.000000000 +0100
@@ -1 +1 @@
-flock -n /tmp/tumblesle_release.lock -c "tumblesle-release --openqa-host
http://openqa.suse.de --group-id 55 --product 'Server' --src
openqa:/var/lib/openqa/factory/ --match '*SP3*Server*x86_64*' --match-hdd
'SLES-12-SP3-x86_64*' -vvvv --post-release-hook
/home/tumblesle/bin/update_jekyll $@" 2>&1 | tee -a
/var/log/tumblesle/tumblesle-release.log
+flock -n /tmp/tumblesle_release.lock -c "tumblesle-release --openqa-host
http://openqa.suse.de --group-id 55 --product 'Server' --src
openqa:/var/lib/openqa/factory/ --match '*SP3*Server*x86_64*' --match-hdd
'SLES-12-SP3-x86_64*' -vvvv --post-release-hook
/home/tumblesle/bin/update_jekyll --seen-maxlen=24 $@" 2>&1 | tee -a
/var/log/tumblesle/tumblesle-release.log
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openqa_review-1.4.1/bin/update_jekyll
new/openqa_review-1.5.0/bin/update_jekyll
--- old/openqa_review-1.4.1/bin/update_jekyll 2017-01-20 14:23:49.000000000
+0100
+++ new/openqa_review-1.5.0/bin/update_jekyll 2017-02-06 18:06:38.000000000
+0100
@@ -1,5 +1,7 @@
#!/bin/sh -e
-old=$(find /srv/www/tumblesle -maxdepth 1 | head -n -3 | grep '[0-9]\+$')
-chmod -R +w ${old}
-rm -rf ${old}
+old=$(find /srv/www/tumblesle -maxdepth 1 | head -n -3 | grep '[0-9]\+$') ||
true
+if [ "$old" != "" ]; then
+ chmod -R +w ${old}
+ rm -rf ${old}
+fi
sudo -u wwwrun /srv/jekyll-source/.git/hooks/post-merge
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openqa_review-1.4.1/openqa_review/browser.py
new/openqa_review-1.5.0/openqa_review/browser.py
--- old/openqa_review-1.4.1/openqa_review/browser.py 2017-01-20
14:23:49.000000000 +0100
+++ new/openqa_review-1.5.0/openqa_review/browser.py 2017-02-06
18:06:38.000000000 +0100
@@ -25,6 +25,11 @@
pass
+class CacheNotFoundError(DownloadError):
+ """content could not retrieved from cache."""
+ pass
+
+
def url_to_filename(url):
"""
Convert URL to a valid, unambigous filename.
@@ -88,7 +93,7 @@
msg = "Request to %s was not successful, file %s not
found" % (url, filename)
log.info(msg)
# as 'load' simulates downloading we also have to simulate
an appropriate error
- raise DownloadError(msg)
+ raise CacheNotFoundError(msg)
else: # pragma: no cover
raise
content = json.loads(raw) if as_json else raw
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openqa_review-1.4.1/openqa_review/openqa_review.py
new/openqa_review-1.5.0/openqa_review/openqa_review.py
--- old/openqa_review-1.4.1/openqa_review/openqa_review.py 2017-01-20
14:23:49.000000000 +0100
+++ new/openqa_review-1.5.0/openqa_review/openqa_review.py 2017-02-06
18:06:38.000000000 +0100
@@ -614,6 +614,7 @@
def add_comment(self, comment):
"""Add a comment to an issue with RPC/REST operations."""
+ log.info("Posting a comment on %s ticket [%s](%s)" % (self.issue_type,
self.bugref, self.bugref_href))
if self.issue_type == 'bugzilla':
self.bugzilla_browser.json_rpc_post('/jsonrpc.cgi',
'Bug.add_comment', {
"id": self.bugid,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/openqa_review-1.4.1/openqa_review/tumblesle_release.py
new/openqa_review-1.5.0/openqa_review/tumblesle_release.py
--- old/openqa_review-1.4.1/openqa_review/tumblesle_release.py 2017-01-20
14:23:49.000000000 +0100
+++ new/openqa_review-1.5.0/openqa_review/tumblesle_release.py 2017-02-06
18:06:38.000000000 +0100
@@ -16,6 +16,10 @@
directory structure is available on. The source, i.e. where to copy assets
from,
can be remote and the path can be specified in an rsync-compatible way, e.g.
"openqa:/var/lib/openqa/factory/"
+
+Notifications over AMQP are sent out if a host has been configured
+appropriately in the configuration file (see config file example). The
+notifications are serialized in JSON strings.
"""
# Python 2 and 3: easiest option
@@ -29,11 +33,13 @@
import fnmatch
import glob
import logging
+import json
import os.path
+import pika
import re
import sys
import time
-from collections import defaultdict
+from collections import defaultdict, deque
from configparser import ConfigParser
from subprocess import check_call
@@ -56,6 +62,8 @@
# example scenario: openSUSE-Tumbleweed-DVD-x86_64-gnome@64bit
whitelist = arm7l-foo,bar@uefi
+#[notification]
+#host = localhost
"""
@@ -101,6 +109,32 @@
log.info("Whitelist content for %s: %s" % (self.args.product,
self.whitelist))
self.release_info_path = os.path.join(self.args.dest,
self.args.release_file)
self.browser = Browser(args, args.openqa_host)
+ if not config.has_section('notification'):
+ return
+ notify_host = config.get('notification', 'host',
fallback='kazhua.suse.de')
+ self.notify_connection =
pika.BlockingConnection(pika.ConnectionParameters(host=notify_host))
+ self.notify_channel = self.notify_connection.channel()
+ self.notify_channel.exchange_declare(exchange='pubsub', type='topic',
passive=True, durable=True)
+ self.notify_topic = 'suse.tumblesle'
+ self.notify_seen = deque(maxlen=args.seen_maxlen)
+
+ def __del__(self):
+ """Cleanup notification objects."""
+ if not hasattr(self, 'notify_connection'):
+ return
+ self.notify_connection.close()
+
+ def notify(self, message, topic='info'):
+ """Send notification over messaging bus."""
+ if not hasattr(self, 'notify_channel'):
+ log.debug("No notification channel enabled, discarding notify.")
+ return
+ body = json.dumps(message)
+ if body in self.notify_seen:
+ log.debug("notification message already sent out recently, not
resending: %s" % body)
+ return
+ self.notify_channel.basic_publish(exchange='pubsub',
routing_key='.'.join([self.notify_topic, topic]), body=body)
+ self.notify_seen.append(body)
def run(self, do_run=True):
"""Continously run while 'do_run' is True, check for last build and
release if satisfying."""
@@ -128,7 +162,7 @@
def is_matching_iso(i):
return 'iso' in i['type'] and 'Staging' not in i['name'] and
re.match(match_re, i['name'])
log.debug("Finding most recent ISO matching regex '%s'" % match_re)
- assets = self.browser.get_json('/api/v1/assets')['assets']
+ assets = self.browser.get_json('/api/v1/assets',
cache=self.args.load)['assets']
isos = [i['name'] for i in assets if is_matching_iso(i)]
return isos
@@ -136,7 +170,7 @@
"""Retrieve jobs for current group by build id, returns dict with
result as keys."""
group_id = int(self.args.group_id)
log.debug("Getting jobs in build %s ..." % build)
- jobs_build =
self.browser.get_json('/api/v1/jobs?state=done&latest=1&build=%s&group_id=%s' %
(build, group_id))['jobs']
+ jobs_build =
self.browser.get_json('/api/v1/jobs?state=done&latest=1&build=%s&group_id=%s' %
(build, group_id), cache=self.args.load)['jobs']
jobs_in_build_product = [i for i in jobs_build if i['group_id'] ==
group_id]
jobs_by_result = defaultdict(list)
for job in jobs_in_build_product:
@@ -215,6 +249,7 @@
new_fixed = sets['released'].difference(sets['last'])
log.info("Regression in new build %s, new failures: %s" %
(build['last'], ', '.join(new_failures)))
log.debug("new fixed: %s" % ', '.join(new_fixed))
+ self.notify({'build': build['last'], 'new_failures':
list(new_failures)}, topic='regression')
# # assuming every job in released_failed is in whitelist
# if len(hard_failed) > previous_hard_failed:
@@ -280,6 +315,7 @@
self.update_symlinks(build_dest)
self.update_release_info()
log.debug("Release DONE")
+ self.notify({'build': self.release_build}, topic='release')
if self.args.post_release_hook:
log.debug("Calling post_release_hook '%s'" %
self.args.post_release_hook)
check_call(self.args.post_release_hook)
@@ -343,6 +379,12 @@
parser.add_argument('--post-release-hook',
help="Specify application path for a post-release hook
which is called after every successful release",
default=None)
+ parser.add_argument('--seen-maxlen', type=int,
+ help="""The length of the 'seen' buffer for
notifications. Any AMQP notification is stored in a FIFO and
+ before sending it is checked if the notification was
already sent out recently with same content.
+ Together with '--sleeptime' the interval under which
the same message would be resent can be configured,
+ e.g. maxlen*sleeptime = minimum time of reappearence
(s)""",
+ default=500)
add_load_save_args(parser)
return parser.parse_args()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openqa_review-1.4.1/setup.py
new/openqa_review-1.5.0/setup.py
--- old/openqa_review-1.4.1/setup.py 2017-01-20 14:23:49.000000000 +0100
+++ new/openqa_review-1.5.0/setup.py 2017-02-06 18:06:38.000000000 +0100
@@ -34,6 +34,11 @@
"humanfriendly",
"requests",
"PyYAML",
+ "pika",
+ "certifi",
+ ],
+ test_require=[
+ "pytest-mock",
],
author="Oliver kurz",
author_email="[email protected]",
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openqa_review-1.4.1/tests/test_tumblesle_release.py
new/openqa_review-1.5.0/tests/test_tumblesle_release.py
--- old/openqa_review-1.4.1/tests/test_tumblesle_release.py 2017-01-20
14:23:49.000000000 +0100
+++ new/openqa_review-1.5.0/tests/test_tumblesle_release.py 2017-02-06
18:06:38.000000000 +0100
@@ -73,6 +73,7 @@
args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'tumblesle/0046_0056_new_release')
args.dest = '/tmp/'
args.post_release_hook = None
+ args.seen_maxlen = 1
# Enable saving and disable loading if you want to add new test data
downloaded from hosts
# args.save = True
# args.save_dir = args.load_dir
@@ -175,3 +176,20 @@
tr = tumblesle_release.TumblesleRelease(args)
tr.check_last_builds()
assert tr.release_build == '0215'
+
+
+def test_notifications_are_sent(args, mocker):
+ with TumblesleDirectory(args) as tmp_dir:
+ config_path = os.path.join(tmp_dir, 'config_file')
+ with open(config_path, 'a') as config:
+ config.write("""
+[notification]
+host = localhost
+""")
+ mocker.patch('pika.BlockingConnection')
+ tr = tumblesle_release.TumblesleRelease(args)
+ tr.one_run()
+ assert '{"build": "0056"}' in tr.notify_seen
+ # this will yield the same message and is therefore not sent out again
+ tr.one_run()
+ assert '{"build": "0056"}' in tr.notify_seen
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openqa_review-1.4.1/tox.ini
new/openqa_review-1.5.0/tox.ini
--- old/openqa_review-1.4.1/tox.ini 2017-01-20 14:23:49.000000000 +0100
+++ new/openqa_review-1.5.0/tox.ini 2017-02-06 18:06:38.000000000 +0100
@@ -51,12 +51,14 @@
deps = -rrequirements.txt
pytest
pytest-cov
+ pytest-mock
[testenv]
commands =
py.test {posargs:-m 'not webtest'}
deps = -rrequirements.txt
pytest
+ pytest-mock
[testenv:webtests]
# This testenv could be called explicitly, e.g. by 'tox -e webtests'.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/openqa_review-1.4.1/update_jekyll
new/openqa_review-1.5.0/update_jekyll
--- old/openqa_review-1.4.1/update_jekyll 2017-01-20 14:23:49.000000000
+0100
+++ new/openqa_review-1.5.0/update_jekyll 1970-01-01 01:00:00.000000000
+0100
@@ -1,7 +0,0 @@
-#!/bin/sh -e
-old=$(find /srv/www/tumblesle -maxdepth 1 | head -n -3 | grep '[0-9]\+$') ||
true
-if [ "$old" != "" ]; then
- chmod -R +w ${old}
- rm -rf ${old}
-fi
-sudo -u wwwrun /srv/jekyll-source/.git/hooks/post-merge
