Jonathan Cave has proposed merging ~jocave/hwcert-jenkins-tools:remove-trello-scripts into hwcert-jenkins-tools:master.
Requested reviews: Canonical Hardware Certification (canonical-hw-cert) For more details, see: https://code.launchpad.net/~jocave/hwcert-jenkins-tools/+git/hwcert-jenkins-tools/+merge/429214 Remove the trello scripts from this repo. They are now hosted in this standalone project: https://github.com/canonical/certification-dashboard-manager -- Your team Canonical Hardware Certification is requested to review the proposed merge of ~jocave/hwcert-jenkins-tools:remove-trello-scripts into hwcert-jenkins-tools:master.
diff --git a/trello-board-manager-desktop.py b/trello-board-manager-desktop.py deleted file mode 100755 index 15c9c00..0000000 --- a/trello-board-manager-desktop.py +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2019 Canonical Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# Written by: -# Taihsiang Ho <[email protected]> -# -# -# The script uses py-trello package 0.9.0. You may want to fetch it from -# source. -# -# This script will move cards between different lanes (-updates and -proposed) - -import sys -import argparse -import logging -import os -import re -import requests -import yaml - -from urllib.parse import urlparse -from trello import TrelloClient - - -repositories = ['proposed', 'updates'] - -repository_promotion_map = { - # repository -> next-repository - 'proposed': 'updates' -} - -codename_map = {'xenial': '16.04', - 'bionic': '18.04', - 'cosmic': '18.10', - 'disco': '19.04', - 'focal': '20.04', - 'groovy': '20.10', - 'hirsute': '21.04'} - -logger = logging.getLogger("trello-board-manager-desktop") - - -def environ_or_required(key): - """Mapping for argparse to supply required or default from $ENV.""" - if os.environ.get(key): - return {'default': os.environ.get(key)} - else: - return {'required': True} - - -def archive_card(card): - logger.info('Archiving old revision: {}'.format(card)) - card.set_closed(True) - - -def move_card(config, lane_name, card): - """Move trello cards according to the current deb repository.""" - logger.info('Card {} in {}'.format(card.name, lane_name)) - m = re.match( - r"(?P<stack>.*?)(?:\s+\-\s+)(?P<package>.*?)(?:\s+\-\s+)" - r"\((?P<version>.*?)\)", card.name) - if m: - arch = config[m.group("stack")]["arch"] - stack = m.group("stack") - codename = m.group("stack").split('-')[0] - - # TODO: you may need to update this mapping in the future when - # the oem image is based on the other distro - if codename == 'oem': - codename = 'bionic' - elif codename == 'oem_focal': - codename = 'focal' - - logger.debug('arch: {}'.format(arch)) - logger.debug('stack: {}'.format(stack)) - logger.debug('codename: {}'.format(codename)) - - for repo in repositories: - logger.debug('Working in repo: {}'.format(repo)) - - jenkins_link = os.environ.get('BUILD_URL', '') - uri = urlparse(jenkins_link) - jenkins_host = '{uri.scheme}://{uri.netloc}/'.format(uri=uri) - logger.debug('jenkins_link: {}'.format(jenkins_link)) - logger.debug('jenkins_host: {}'.format(jenkins_host)) - # TODO: we could merge main and universe repositories from the source - # jenkins jobs - # linux-oem is in universe rather than main - if 'oem-osp1' in stack and not codename == 'xenial': - text_template = '{}/job/cert-package-data/lastSuccessfulBuild/artifact/{}-universe-{}-{}.json' - else: - # packages of generic kernels - # projects using these kernels: - # stock images - # oem image - xenial - # oem images - shipped with oem-4.13 - # argos dgx-1/dgx-station images - text_template = '{}/job/cert-package-data/lastSuccessfulBuild/artifact/{}-main-{}-{}.json' - - package_json_url = text_template.format(jenkins_host, - codename, arch, repo) - response = requests.get(url=package_json_url) - pkg_data = response.json() - - # stack_version_full, svf, for example 4.4.0.150.158 - if 'oem' in stack and '5.6' in card.name: - svf = pkg_data['linux-oem-20_04'] - elif 'oem' in stack and '5.10' in card.name: - svf = pkg_data['linux-oem-20_04b'] - elif 'oem' in stack and '5.13' in card.name: - svf = pkg_data['linux-oem-20_04c'] - elif 'oem' in stack and '5.14' in card.name: - svf = pkg_data['linux-oem-20_04d'] - else: - svf = pkg_data['linux-generic'] - - if 'hwe' in stack: - svf = re.match(r'\d+.\d+.\d+.\d+.\d+', - pkg_data['linux-generic-hwe-' + codename_map[codename].replace('.', '_')]).group(0) - # I only want 4_4_0-150 - sv = svf[:svf.rfind('.')].replace('.', '-') - stack_version = sv.replace('-', '_', 2) - if 'oem-osp1' in stack: - deb_kernel_image = 'linux-image-' + stack_version + '-oem-osp1' - elif "oem" in stack: - deb_kernel_image = 'linux-image-' + stack_version + '-oem' - else: - deb_kernel_image = 'linux-image-' + stack_version + '-generic' - - logger.debug('stack_version: {}'.format(stack_version)) - logger.debug('deb_kernel_image: {}'.format(deb_kernel_image)) - - deb_version = pkg_data[deb_kernel_image] - - ori = next_repo = repository_promotion_map.get(lane_name) - - logger.debug('deb_version: {}'.format(deb_version)) - logger.debug('next_repo: {}'.format(next_repo)) - logger.debug('m.group("stack"): ' - '{} {}'.format(m.group("stack"), - type(m.group("stack")))) - logger.debug('m.group("package"): ' - '{} {}'.format(m.group("package"), - type(m.group("package")))) - logger.debug('m.group("version"): ' - '{} {}'.format(m.group("version"), - type(m.group("version")))) - - if (repo == lane_name and deb_version != m.group("version")): - archive_card(card) - continue - if (repo == next_repo and - deb_kernel_image == m.group("package") and - deb_version == m.group("version")): - for l in card.board.open_lists(): - if ori.capitalize() == l.name: - msg = 'Moving the card {} to {}'.format(card.name, - l.name) - logger.debug(msg) - card.change_list(l.id) - return - - -def load_config(configfile): - if not configfile: - return [] - try: - data = yaml.safe_load(configfile) - except (yaml.parser.ParserError, yaml.scanner.ScannerError): - print('ERROR: Error parsing', configfile.name) - sys.exit(1) - return data - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--key', help="Trello API key", - **environ_or_required('TRELLO_API_KEY')) - parser.add_argument('--token', help="Trello OAuth token", - **environ_or_required('TRELLO_TOKEN')) - parser.add_argument('--board', help="Trello board identifier", - **environ_or_required('TRELLO_BOARD')) - parser.add_argument('--debug', help="Enable the debug mode", - action="store_true", default=False) - parser.add_argument('config', help="snaps configuration", - type=argparse.FileType()) - args = parser.parse_args() - - format_str = "[ %(funcName)s() ] %(message)s" - if args.debug: - logging.basicConfig(format=format_str, level=logging.DEBUG) - else: - logging.basicConfig(format=format_str) - - client = TrelloClient(api_key=args.key, token=args.token) - board = client.get_board(args.board) - - config = load_config(args.config) - - for lane in board.list_lists(): - lane_name = lane.name.lower() - if lane_name in repositories: - for card in lane.list_cards(): - try: - move_card(config, lane_name, card) - except Exception: - logger.warning("WARNING", exc_info=True) - continue - - -if __name__ == "__main__": - main() diff --git a/trello-board-manager.py b/trello-board-manager.py deleted file mode 100755 index fa715d9..0000000 --- a/trello-board-manager.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2017 Canonical Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# Written by: -# Sylvain Pineau <[email protected]> - -import argparse -import logging -import os -import re -import requests - -from trello import TrelloClient -from yaml import safe_load -from yaml.parser import ParserError - - -channel_promotion_map = { - # channel -> next-channel - 'edge': 'beta', - 'beta': 'candidate', - 'candidate': 'stable', - 'stable': 'stable', -} - - -def environ_or_required(key): - """Mapping for argparse to supply required or default from $ENV.""" - if os.environ.get(key): - return {'default': os.environ.get(key)} - else: - return {'required': True} - - -def archive_card(card): - print('Archiving old revision:', card) - card.set_closed(True) - - -def move_card(config, lane_name, card): - """Move trello cards according to the snap current channel.""" - print(card.name) - m = re.match( - r"(?P<snap>.*?)(?:\s+\-\s+)(?P<version>.*?)(?:\s+\-\s+)" - r"\((?P<revision>.*?)\)(?:\s+\-\s+\[(?P<track>.*?)\])?", card.name) - if m: - arch = config[m.group("snap")]["arch"] - headers = { - 'Snap-Device-Series': '16', - 'Snap-Device-Architecture': arch, - 'Snap-Device-Store': config[m.group("snap")]["store"], - } - req = requests.get( - 'https://api.snapcraft.io/v2/' - 'snaps/info/{}'.format(m.group("snap")), - headers=headers) - json_resp = req.json() - track = m.group("track") - if not track: - track = 'latest' - for channel_info in json_resp["channel-map"]: - if (channel_info["channel"]["track"] == track and - channel_info["channel"]["architecture"] == arch): - risk = channel_info["channel"]["risk"] - try: - version = channel_info['version'] - revision = str(channel_info['revision']) - except KeyError: - continue - ori = next_risk = channel_promotion_map[lane_name] - # If the snap with this name, in this channel is a - # differet revision, then this is old so archive it - if (risk == lane_name and - revision != m.group("revision")): - archive_card(card) - continue - if ( - risk == next_risk and - version == m.group("version") and - revision == m.group("revision") - ): - for l in card.board.open_lists(): - if ori.capitalize() == l.name: - card.change_list(l.id) - return - - -def main(): - logger = logging.getLogger("trello-board-manager") - parser = argparse.ArgumentParser() - parser.add_argument('--key', help="Trello API key", - **environ_or_required('TRELLO_API_KEY')) - parser.add_argument('--token', help="Trello OAuth token", - **environ_or_required('TRELLO_TOKEN')) - parser.add_argument('--board', help="Trello board identifier", - **environ_or_required('TRELLO_BOARD')) - parser.add_argument('config', help="snaps configuration", - type=argparse.FileType()) - args = parser.parse_args() - client = TrelloClient(api_key=args.key, token=args.token) - board = client.get_board(args.board) - try: - config = safe_load(args.config) - except ParserError: - raise SystemExit('Error parsing %s' % args.config.name) - for lane in board.list_lists(): - lane_name = lane.name.lower() - if lane_name in channel_promotion_map.keys(): - for card in lane.list_cards(): - try: - move_card(config, lane_name, card) - except Exception: - logger.warn("WARNING", exc_info=True) - continue - - -if __name__ == "__main__": - main() diff --git a/trello-board-updater-desktop.py b/trello-board-updater-desktop.py deleted file mode 100755 index 719fc92..0000000 --- a/trello-board-updater-desktop.py +++ /dev/null @@ -1,623 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2019 Canonical Ltd. -# All rights reserved. -# -# Written by: -# Taihsiang Ho <[email protected]> -# Sylvain Pineau <[email protected]> -# -# -# The script uses py-trello package 0.9.0. You may want to fetch it from -# source. -# -# This script will create or update corresponding cards for each kernel and -# SUTs -# -import argparse -import importlib -import os -import re -import requests -import sys -import yaml -import logging -import collections - -import unittest -import json -import trello - -from urllib.parse import urlparse -from datetime import datetime -from trello import TrelloClient -from trello.exceptions import ResourceUnavailable - -from unittest.mock import MagicMock - -tbu = importlib.import_module("trello-board-updater") - -format_str = "[ %(funcName)s() ] %(message)s" -logging.basicConfig(level=logging.INFO, format=format_str) - - -def run(args, board, c3_link, jenkins_link): - - kernel_stack = args.series - if args.sru_type == 'stock-hwe': - kernel_stack = args.series + '-hwe' - - # The current oem stack - # For now it is bionic with linux-oem kernel - # TODO: update the if statement when oem-osp1 is delivered - if kernel_stack == 'bionic' and 'oem' in args.kernel: - if 'osp1' in args.kernel: - kernel_stack = 'oem-osp1' - else: - kernel_stack = 'oem_bionic' - if kernel_stack == 'focal' and 'oem' in args.kernel: - kernel_stack = 'oem_focal' - - uri = urlparse(jenkins_link) - jenkins_host = '{uri.scheme}://{uri.netloc}/'.format(uri=uri) - - # TODO: we could merge main and universe repositories from the source - # jenkins jobs - # linux-raspi is in universe for bionic - if args.series == 'bionic' and 'raspi' in args.kernel: - package_json_name_template = '{}-universe-{}-proposed.json' - else: - # packages of generic kernels - # projects using these kernels: - # stock images - # oem image - xenial - # oem images - shipped with oem-4.13 - # argos dgx-1/dgx-station images - package_json_name_template = '{}-main-{}-proposed.json' - - package_json_url_template = '{}/job/cert-package-data/'\ - 'lastSuccessfulBuild/artifact/' + \ - package_json_name_template - - package_json_url = package_json_url_template.format(jenkins_host, - args.series, - args.arch) - logging.info('package json url: {}'.format(package_json_url)) - response = requests.get(url=package_json_url) - package_data = response.json() - - # Since bionic-oem and bionic-oem-osp1 are retired, we should move oem item to generic card. - # In order to distinguish them, we add a prefix to the item name. - if 'oem' in args.kernel and 'oem' in args.sru_type and args.series == 'bionic': - print(kernel_stack,args.kernel) - kernel_stack = 'bionic-hwe' - if 'osp1' in args.kernel: - args.name = 'oem-osp1-'+ args.name + '-hwe' - else: - args.name = 'oem-'+ args.name + '-hwe' - #since some focal-oem machines will upgrade to 5.8-hwe kernel, therefore we should move those item to focal-hwe card. - if 'hwe' in args.kernel and 'oem' in args.sru_type and args.series == 'focal': - print(kernel_stack,args.kernel) - kernel_stack = 'focal-hwe' - args.name = 'oem-'+ args.name + '-hwe' - - # linux deb version - # e.g. linux-generic-hwe-16.04 which version is 4.15.0.50.71 - dlv = package_data[args.kernel].split('.') - dlv_short = dlv[0] + '_' + dlv[1] + '_' + dlv[2] + '-' + dlv[3] - logging.info("linux deb version: {}".format(dlv)) - logging.info("linux deb version (underscores): {}".format(dlv_short)) - kernel_suffix = kernel_stack.split('_')[0] - if args.sru_type == 'stock' or args.sru_type == 'stock-hwe': - # for stock images, it always uses generic kernels - # - # GA example: - # linux-generic --> - # linux-image-4_15_0-55-generic (4.15.0.55.57) - # - # hwe stack example: - # linux-generic-hwe-18_04 --> - # linux-image-5_0_0-21-generic (5.0.0-21.22~18.04.1) - kernel_suffix = 'generic' - elif args.sru_type == 'oem' and "xenial" in args.series: - # very special case: oem xenial images - # oem xenial 4.4 kernel is using generic kernel, besides, - # some oem images are delivered as xenial + oem-4.13 - # when time goes by, it is updated to be xenial + generic xenial hwe - # - # this if condition includes the dgx-1 and dgx-station images - # - # TODO: we may need to add more conditions when more oem images - # is updated to use generic kernel - kernel_suffix = 'generic' - elif args.sru_type == 'oem' and "bionic" in args.series: - # bionic oem image has started using generic kernel. - # bionic-oem and bionic-oem-osp1 will upgrade to 5.4 kernel. - kernel_suffix = 'generic' - elif args.sru_type == 'oem' and "focal" in args.series and 'hwe' in args.kernel: - # some focal-oem machines will upgrade to 5.8-hwe kernel. - kernel_suffix = 'generic' - - # For raspi kernels - if 'raspi' in args.kernel: - if "xenial" in args.series: - kernel_suffix = 'raspi2' - if "bionic" in args.series: - if "hwe" in args.kernel: - kernel_suffix = 'raspi' - else: - kernel_suffix = 'raspi2' - else: - kernel_suffix = 'raspi' - kernel_stack = kernel_stack + '-' + kernel_suffix - - logging.info("kernel_suffix: {}".format(kernel_suffix)) - - deb_kernel_image = 'linux-image-' + dlv_short + '-' + kernel_suffix - - deb_version = package_data[deb_kernel_image] - pattern = "{} - {} - \({}\)".format( - re.escape(kernel_stack), - re.escape(deb_kernel_image), - re.escape(deb_version)) - card = tbu.search_card(board, pattern) - config = tbu.load_config(args.config, None) - expected_tests = config.get(kernel_stack, {}).get('expected_tests', []) - - logging.info('SRU type: {}'.format(args.sru_type)) - logging.info('series: {}'.format(args.series)) - logging.info("kernel_stack: {}".format(kernel_stack)) - logging.info("deb_kernel_image: {}".format(deb_kernel_image)) - logging.info("deb_version: {}".format(deb_version)) - logging.info("expected_tests and SUTs: {}".format(expected_tests)) - - lanes = ['Proposed', 'Updates'] - - if not card: - lane = None - for l in board.open_lists(): - # TODO: not a reasonable condition, use better one later - if l.name == lanes[0] and \ - package_data[deb_kernel_image] == deb_version: - lane = l - break - if lane: - print("No target card was found. Create an new one...") - card = lane.add_card('{} - {} - ({})'.format(kernel_stack, - deb_kernel_image, - deb_version)) - else: - print('No target card and lane was found. Give up to create an ' - 'new card.') - sys.exit(1) - if not args.cardonly: - summary = '**[TESTFLINGER] {} {} {} {} ({})**\n---\n\n'.format( - args.name, args.arch, args.kernel, deb_kernel_image, deb_version) - summary += '- Jenkins build details: {}\n'.format(jenkins_link) - summary += '- Full results at: {}\n\n```\n'.format(c3_link) - summary_data = args.summary.read() - summary += summary_data - summary += '\n```\n' - comment = card.comment(summary) - comment_link = "{}#comment-{}".format(card.url, comment['id']) - else: - summary_data = "" - checklist = tbu.find_or_create_checklist(card, 'Testflinger', expected_tests) - - # Need to test different arch on the same device, separate the results - # TODO: We should support different arch for all devices, not just raspi - if 'raspi' in args.kernel: - args.name = args.name + '-' + args.arch - - if args.cardonly: - item_content = "{} ({})".format(args.name, 'In progress') - else: - item_content = "[{}]({}) ({})".format( - args.name, comment_link, datetime.utcnow().isoformat()) - if jenkins_link: - item_content += " [[JENKINS]({})]".format(jenkins_link) - if c3_link: - item_content += " [[C3]({})]".format(c3_link) - elif not args.cardonly: - # If there was no c3_link, it's because the submission failed - tbu.attach_labels(board, card, ['TESTFLINGER CRASH']) - - # debug message - logging.info('checklist: {}'.format(checklist)) - logging.info('item_content: {}'.format(item_content)) - if not tbu.change_checklist_item( - checklist, args.name, item_content, - checked=tbu.no_new_fails_or_skips(summary_data)): - checklist.add_checklist_item(item_content) - - if not [c for c in card.fetch_checklists() if c.name == 'Sign-Off']: - checklist = tbu.find_or_create_checklist(card, 'Sign-Off') - checklist.add_checklist_item('Ready for ' + lanes[0], True) - checklist.add_checklist_item('Ready for ' + lanes[1]) - checklist.add_checklist_item('Can be Archived') - checklist = tbu.find_or_create_checklist(card, 'Revisions') - rev = '{} ({})'.format(deb_version, args.arch) - if rev not in [item['name'] for item in checklist.items]: - checklist.add_checklist_item(rev) - - # a read trello card object, useful for testing - k_deb_card = collections.namedtuple("KernelDeb", ["kernel_stack", - "deb_kernel_image", - "deb_version", - "expected_tests", - "sut"]) - # card title - k_deb_card.kernel_stack = kernel_stack - k_deb_card.deb_kernel_image = deb_kernel_image - k_deb_card.deb_version = deb_version - # card content: SUTs - k_deb_card.expected_tests = expected_tests - k_deb_card.sut = args.name - - return k_deb_card - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--key', help="Trello API key", - **tbu.environ_or_required('TRELLO_API_KEY')) - parser.add_argument('--token', help="Trello OAuth token", - **tbu.environ_or_required('TRELLO_TOKEN')) - parser.add_argument('--board', help="Trello board identifier", - **tbu.environ_or_required('TRELLO_BOARD')) - parser.add_argument('--config', help="Pool configuration", - type=argparse.FileType()) - parser.add_argument('-a', '--arch', help="deb architecture", - required=True) - parser.add_argument('-t', '--sru-type', - help="SRU type, stock or oem etc.", required=True) - parser.add_argument('-s', '--series', - help="series code name, e.g. xenial etc.", - required=True) - parser.add_argument('-n', '--name', help="SUT name", required=True) - parser.add_argument('-k', '--kernel', help="kernel type", required=True) - # TODO: for now it is not required because I want to make it backward - # compatible. If we could batch update the corresponding jenkins jobs then - # we could make this argument required. - parser.add_argument('-q', '--queue', help="kernel type", default="") - parser.add_argument('summary', help="test results summary", - type=argparse.FileType()) - parser.add_argument("--cardonly", help="Only create an empty card", - action="store_true") - args = parser.parse_args() - client = TrelloClient(api_key=args.key, token=args.token) - board = client.get_board(args.board) - c3_link = os.environ.get('C3LINK', '') - jenkins_link = os.environ.get('BUILD_URL', '') - - run(args, board, c3_link, jenkins_link) - - -class TestTrelloUpdaterKernelDebSRU(unittest.TestCase): - - def _request_get(self): - return requests.models.Response() - - def _get_package_data(self): - with open(self.packages_info) as f: - data = json.load(f) - - return data - - def _get_cards(self, board, card_id, name): - card = trello.card.Card(board, card_id, name=name) - return [card] - - def _mock_factory(self, jenkins_job_template, card_name, packages_info): - parser = argparse.ArgumentParser() - args = parser.parse_args([]) - args.__dict__.update(jenkins_job_template) - - self.args = args - self.board = trello.board.Board("fake_board") - self.c3_link = "fake_c3_link" - self.jenkins_link = "fake_jenkins_link" - self.packages_info = packages_info - - requests.get = MagicMock(return_value=self._request_get()) - requests.models.Response.json = MagicMock( - side_effect=self._get_package_data) - trello.board.Board.get_cards = MagicMock(return_value=self._get_cards( - self.board, - 9999, - card_name)) - trello.board.Card.comment = MagicMock(return_value="fake_comment") - trello.board.Card.fetch_checklists = MagicMock( - return_value=[]) - mock_checklist = MagicMock() - mock_checklist.add_checklist_item = print - - trello.board.Card.add_checklist = MagicMock( - return_value=mock_checklist) - - def setUp(self): - self.debs_yaml_stream = open('./data/debs.yaml') - self.summary_stream = open('./data/raw_summary') - - def tearDown(self) -> None: - self.debs_yaml_stream.close() - self.summary_stream.close() - - def test_stock_xenial_4_4_kernel_stack(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'xenial-desktop-201606-22344', - 'arch': 'amd64', - 'kernel': 'linux-generic', - 'series': 'xenial', - 'sru_type': 'stock', - 'queue': '', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "xenial - linux-image-4_4_0-167-generic - (4.4.0-167.196)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_xenial-main-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - self.assertEqual(kdeb_card.kernel_stack, "xenial") - - def test_stock_xenial_4_15_kernel_stack(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'xenial-hwe-desktop-201606-22344', - 'arch': 'amd64', - 'kernel': 'linux-generic-hwe-16_04', - 'series': 'xenial', - 'sru_type': 'stock', - 'queue': '', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "xenial-hwe - linux-image-4_15_0-66-generic " \ - "- (4.15.0-66.75~16.04.1)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_xenial-main-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - self.assertEqual(kdeb_card.kernel_stack, "xenial-hwe") - - def test_stock_bionic_4_15_kernel_stack(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'bionic-desktop-201606-22344', - 'arch': 'amd64', - 'kernel': 'linux-generic', - 'series': 'bionic', - 'sru_type': 'stock', - 'queue': '', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "bionic - linux-image-4_15_0-67-generic - (4.15.0-67.76)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_bionic-main-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - self.assertEqual(kdeb_card.kernel_stack, "bionic") - - def test_oem_xenial_4_4_kernel_stack(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'xenial-desktop-201610-25144', - 'arch': 'amd64', - 'kernel': 'linux-generic', - 'series': 'xenial', - 'sru_type': 'oem', - 'queue': '', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "xenial - linux-image-4_4_0-167-generic - (4.4.0-167.196)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_xenial-main-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - self.assertEqual(kdeb_card.kernel_stack, "xenial") - - def test_oem_xenial_4_15_kernel_stack(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'xenial-hwe-desktop-201802-26107', - 'arch': 'amd64', - 'kernel': 'linux-generic-hwe-16_04', - 'series': 'xenial', - 'sru_type': 'oem', - 'queue': '', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "xenial-hwe - linux-image-4_15_0-66-generic " \ - "- (4.15.0-66.75~16.04.1)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_xenial-main-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - self.assertEqual(kdeb_card.kernel_stack, "xenial-hwe") - - def test_oem_bionic_4_15_kernel_stack(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'bionic-desktop-201802-26107', - 'arch': 'amd64', - 'kernel': 'linux-oem', - 'series': 'bionic', - 'sru_type': 'oem', - 'queue': '', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "oem - linux-image-4_15_0-1059-oem - (4.15.0-1059.68)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_bionic-main-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - self.assertEqual(kdeb_card.kernel_stack, "oem") - - def test_oem_osp1_bionic_4_15_kernel_stack(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'bionic-desktop-201906-27089', - 'arch': 'amd64', - 'kernel': 'linux-oem-osp1', - 'series': 'bionic', - 'sru_type': 'oem', - 'queue': '', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "oem-osp1 - linux-image-5_0_0-1025-oem-osp1 " \ - "- (5.0.0-1025.28)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_bionic-universe-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - self.assertEqual(kdeb_card.kernel_stack, "oem-osp1") - - def test_argos_dgx_station_xenial_4_4(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'xenial-desktop-201711-25989', - 'arch': 'amd64', - 'kernel': 'linux-generic', - 'series': 'xenial', - 'sru_type': 'oem', - 'queue': 'argos-201711-25989', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "xenial - linux-image-4_4_0-167-generic - (4.4.0-167.196)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_xenial-main-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - target_sut = "201711-25989-dgx-station" - target_sut_kdc_id = kdeb_card.expected_tests.index(target_sut) - - self.assertEqual(kdeb_card.kernel_stack, "xenial") - self.assertEqual(kdeb_card.sut, target_sut) - self.assertEqual(kdeb_card.expected_tests[target_sut_kdc_id], - target_sut) - - def test_argos_dgx_1_xenial_4_4(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'xenial-server-201802-26098', - 'arch': 'amd64', - 'kernel': 'linux-generic', - 'series': 'xenial', - 'sru_type': 'oem', - 'queue': 'argos-201802-26098', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "xenial - linux-image-4_4_0-167-generic - (4.4.0-167.196)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_xenial-main-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - target_sut = "201802-26098-dgx-1" - target_sut_kdc_id = kdeb_card.expected_tests.index(target_sut) - - self.assertEqual(kdeb_card.kernel_stack, "xenial") - self.assertEqual(kdeb_card.sut, target_sut) - self.assertEqual(kdeb_card.expected_tests[target_sut_kdc_id], - target_sut) - - def test_argos_dgx_1_xenial_hwe(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'xenial-hwe-server-201802-26098', - 'arch': 'amd64', - 'kernel': 'linux-generic-hwe-16_04', - 'series': 'xenial', - 'sru_type': 'oem', - 'queue': 'argos-201802-26098', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "xenial-hwe - linux-image-4_15_0-66-generic " \ - "- (4.15.0-66.75~16.04.1)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_xenial-main-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - target_sut = "201802-26098-dgx-1" - target_sut_kdc_id = kdeb_card.expected_tests.index(target_sut) - - self.assertEqual(kdeb_card.kernel_stack, "xenial-hwe") - self.assertEqual(kdeb_card.sut, target_sut) - self.assertEqual(kdeb_card.expected_tests[target_sut_kdc_id], - target_sut) - - def test_oem_focal_5_6_kernel_stack(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'focal-desktop-xps13-9310-c1', - 'arch': 'amd64', - 'kernel': 'linux-oem', - 'series': 'focal', - 'sru_type': 'oem', - 'queue': '', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "oem_focal - linux-image-oem-20_04 - (5.6.0.1034.30)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_focal-main-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - self.assertEqual(kdeb_card.kernel_stack, "oem") - - def test_stock_focal_5_4_kernel_stack(self): - jenkins_job_template = { - 'cardonly': False, - 'name': 'focal-desktop-inspiron20-3064', - 'arch': 'amd64', - 'kernel': 'linux-generic', - 'series': 'focal', - 'sru_type': 'stock', - 'queue': '', - 'config': self.debs_yaml_stream, - 'summary': self.summary_stream - } - card_name = "focal - linux-image-5_4_0-54-generic - (5.4.0-54.60)" - - self._mock_factory(jenkins_job_template, card_name, - "./data/deb-package_focal-main-amd64.json") - - kdeb_card = run(self.args, self.board, self.c3_link, self.jenkins_link) - - self.assertEqual(kdeb_card.kernel_stack, "focal") - - -if __name__ == "__main__": - main() diff --git a/trello-board-updater-image-testing.py b/trello-board-updater-image-testing.py deleted file mode 100755 index ca97b92..0000000 --- a/trello-board-updater-image-testing.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019-2020 Canonical Ltd. -# All rights reserved. -# -# Written by: -# Paul Larson <[email protected]> -# Sylvain Pineau <[email protected]> - -import argparse -import importlib -import os -import re - -from datetime import datetime -from trello import TrelloClient - -tbu = importlib.import_module("trello-board-updater") -tbm = importlib.import_module("trello-board-manager") - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--key', help="Trello API key", - **tbu.environ_or_required('TRELLO_API_KEY')) - parser.add_argument('--token', help="Trello OAuth token", - **tbu.environ_or_required('TRELLO_TOKEN')) - parser.add_argument('--board', help="Trello board identifier", - **tbu.environ_or_required('TRELLO_BOARD')) - parser.add_argument('-n', '--name', help="SUT device name", required=True) - parser.add_argument('-i', '--image', help="image name", required=True) - parser.add_argument('-c', '--channel', help="image name", default="stable") - parser.add_argument('-v', '--version', help="snap version", required=True) - parser.add_argument('summary', help="test results summary", - type=argparse.FileType()) - args = parser.parse_args() - client = TrelloClient(api_key=args.key, token=args.token) - board = client.get_board(args.board) - c3_link = os.environ.get('C3LINK', '') - jenkins_link = os.environ.get('BUILD_URL', '') - pattern = "{} - {} - {}".format( - re.escape(args.image), - re.escape(args.channel), - re.escape(args.version)) - # First, see if this exact card already exists - card = tbu.search_card(board, pattern) - - # If not, see if there's an older one for this image - if not card: - pattern = "{} - {} - .*".format(re.escape(args.image), args.channel) - card = tbu.search_card(board, pattern) - if card: - tbm.archive_card(card) - # If we get here, then either we just archived the old card, or - # it didn't exist. We need to create it either way - lane = None - for l in board.open_lists(): - if l.name == args.channel: - lane = l - break - if not lane: - lane = board.add_list(args.channel) - card = lane.add_card('{} - {} - {}'.format( - args.image, args.channel, args.version)) - summary = '**[TESTFLINGER] {} {} {}**\n---\n\n'.format( - args.name, args.image, args.version) - summary += '- Jenkins build details: {}\n'.format(jenkins_link) - summary += '- Full results at: {}\n\n```\n'.format(c3_link) - summary_data = args.summary.read() - summary += summary_data - summary += '\n```\n' - comment = card.comment(summary) - comment_link = "{}#comment-{}".format(card.url, comment['id']) - checklist = tbu.find_or_create_checklist(card, 'Testflinger') - item_content = "[{}]({}) ({})".format( - args.name, comment_link, datetime.utcnow().isoformat()) - if jenkins_link: - item_content += " [[JENKINS]({})]".format(jenkins_link) - if c3_link: - item_content += " [[C3]({})]".format(c3_link) - - if not tbu.change_checklist_item( - checklist, args.name, item_content, - checked=tbu.no_new_fails_or_skips(summary_data)): - checklist.add_checklist_item( - item_content, checked=tbu.no_new_fails_or_skips(summary_data)) - - -if __name__ == "__main__": - main() diff --git a/trello-board-updater.py b/trello-board-updater.py deleted file mode 100755 index 9d9af50..0000000 --- a/trello-board-updater.py +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2017 Canonical Ltd. -# All rights reserved. -# -# Written by: -# Sylvain Pineau <[email protected]> - -import argparse -import os -import re -import requests -import sys -import time -import yaml - -from datetime import datetime -from trello import TrelloClient -from trello.exceptions import ResourceUnavailable -from mailtool import send_mail - - -def environ_or_required(key): - if os.environ.get(key): - return {'default': os.environ.get(key)} - else: - return {'required': True} - - -def search_card(board, query, card_filter="open"): - for card in board.get_cards(card_filter=card_filter): - if re.match(query, card.name): - return card - - -def find_or_create_checklist(card, checklist_name, items=[]): - existing_checklists = card.fetch_checklists() - checklist = None - for c in existing_checklists: - if c.name == checklist_name: - checklist = c - break - if not checklist: - print("Creating checklist: {}".format(checklist_name)) - checklist = card.add_checklist(checklist_name, []) - for item in items: - checklist.add_checklist_item(item + ' (NO RESULTS)') - return checklist - - -def change_checklist_item(checklist, name, content, checked=False): - """Attempt to rename and change details on a checklist item - - Return True if the item is located and changes are made - Return False if no matching item was found in the specified checklist - - Since some cards have multiple results checklists, a False return - may be expected in many cases - """ - for item in checklist.items: - if ( - name + ' ' in item.get('name') or - name + ']' in item.get('name') - ): - checklist.rename_checklist_item(item.get('name'), content) - checklist.set_checklist_item(content, checked) - return True - else: - return False - - -def no_new_fails_or_skips(summary_data): - """Check summary data for new fails or skips - - Return True if there are no new fails or skips detected and if - more than 1 test actually ran as a sanity check - """ - return ("No new failed or skipped tests" in summary_data and - "WARNING: Very small number of total tests" not in summary_data) - - -def load_config(configfile, snapname): - if not configfile: - return [] - try: - data = yaml.safe_load(configfile) - except (yaml.parser.ParserError, yaml.scanner.ScannerError): - print('ERROR: Error parsing', configfile.name) - sys.exit(1) - return data - - -def attach_labels(board, card, label_list): - for labelstr in label_list: - for label in board.get_labels(): - if label.name == labelstr: - labels = card.labels or [] - if label not in labels: - # Avoid crash if checking labels fails to find it - try: - card.add_label(label) - except ResourceUnavailable: - pass - break - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--key', help="Trello API key", - **environ_or_required('TRELLO_API_KEY')) - parser.add_argument('--token', help="Trello OAuth token", - **environ_or_required('TRELLO_TOKEN')) - parser.add_argument('--board', help="Trello board identifier", - **environ_or_required('TRELLO_BOARD')) - parser.add_argument('--config', help="Snaps configuration", - type=argparse.FileType()) - parser.add_argument('-a', '--arch', help="snap architecture", - required=True) - parser.add_argument('-b', '--brandstore', help="brand store identifier", - default='ubuntu') - parser.add_argument('-e', '--email', help="Email recipients") - parser.add_argument('-n', '--name', help="SUT name", required=True) - parser.add_argument('-s', '--snap', help="snap name", required=True) - parser.add_argument('-v', '--version', help="snap version", required=True) - parser.add_argument('-r', '--revision', help="snap revision", - required=True) - parser.add_argument('-c', '--channel', help="snap channel", required=True) - parser.add_argument('-t', '--track', help="snap track", required=True) - parser.add_argument('summary', help="test results summary", - type=argparse.FileType()) - parser.add_argument("--cardonly", help="Only create an empty card", - action="store_true") - args = parser.parse_args() - client = TrelloClient(api_key=args.key, token=args.token) - board = client.get_board(args.board) - track = args.track.replace('__track__', '') - c3_link = os.environ.get('C3LINK', '') - jenkins_link = os.environ.get('BUILD_URL', '') - pattern = "{} - {} - \({}\).*{}".format( - re.escape(args.snap), - re.escape(args.version), - args.revision, - re.escape(track)) - card = search_card(board, pattern) - config = load_config(args.config, args.snap) - snap_labels = config.get(args.snap, {}).get('labels', []) - # Try to find the right card using other arch revision numbers - # i.e. We could be recording a result for core rev 111 on armhf, but we - # want to record it against the core rev 110 results with amd64, so find - # that card by finding the arch/rev from the store to use in the search - if not card: - rev_list = dict() - headers = { - 'Snap-Device-Series': '16', - 'Snap-Device-Store': args.brandstore, - } - try: - json = requests.get( - 'https://api.snapcraft.io/v2/' - 'snaps/info/{}'.format(args.snap), - headers=headers).json() - except Exception: - # Sometimes either the request is bad or it returns something that - # .json() doesn't like. If it's a short-term issue, maybe we can - # give it one more chance to pass - print("WARNING: Bad store request, retrying...") - time.sleep(30) - json = requests.get( - 'https://api.snapcraft.io/v2/' - 'snaps/info/{}'.format(args.snap), - headers=headers).json() - # store_track is used for searching for the right track name in the - # store, which would be latest if nothing is defined - # track is used for search in the cards, which will be either the - # default_track, the defined track for the run, or empty for 'latest' - track = config.get(args.snap, {}).get('default_track', track) - store_track = track or "latest" - for channel_info in json['channel-map']: - try: - if channel_info['version'] != args.version: - continue - if (channel_info["channel"]["track"] == store_track and - channel_info["channel"]["risk"] == args.channel): - arch = channel_info["channel"]["architecture"] - rev_list[arch] = channel_info['revision'] - except KeyError: - continue - # Try to find the right revision for the right architecture when a - # channel follows the next one meaning nothing was pushed to it, e.g: - # - # channels: - # stable: 0.7.1 2018-12-11 (329) 387MB - - # candidate: ↑ - # beta: ↑ - if not rev_list: - for risk in ['edge', 'beta', 'candidate', 'stable']: - for channel_info in json['channel-map']: - try: - if channel_info['version'] != args.version: - continue - if channel_info["channel"]["track"] == store_track: - if channel_info["channel"]["risk"] == risk: - arch = channel_info["channel"]["architecture"] - rev_list[arch] = channel_info['revision'] - except KeyError: - continue - if rev_list: - break - for rev in rev_list.values(): - pattern = "{} - {} - \({}\).*{}".format( - re.escape(args.snap), - re.escape(args.version), - rev, - re.escape(track)) - card = search_card(board, pattern) - if card: - # Prefer amd64 rev in card title - if args.arch == 'amd64': - if track: - card.set_name('{} - {} - ({}) - [{}]'.format( - args.snap, args.version, args.revision, - track)) - else: - card.set_name('{} - {} - ({})'.format( - args.snap, args.version, args.revision)) - break - print('Using card: {} ({})'.format(card.name, card.short_url)) - # Create the card in the right lane, since we still didn't find it - # We only want one card for all architectures, so use the revision - # declared for the default arch in snaps.yaml - if not card: - default_arch = config.get(args.snap, {}).get('arch', args.arch) - default_rev = rev_list.get(default_arch, args.revision) - channel = args.channel.capitalize() - lane = None - # Use the default_track if there is one, else use track name specified - track = config.get(args.snap, {}).get('default_track', track) - for l in board.open_lists(): - if channel == l.name: - lane = l - break - if lane: - if track: - card = lane.add_card('{} - {} - ({}) - [{}]'.format( - args.snap, args.version, default_rev, track)) - else: - card = lane.add_card('{} - {} - ({})'.format( - args.snap, args.version, default_rev)) - print('No card found!') - print('Creating card: {} ({})'.format(card.name, card.short_url)) - if args.email: - msg_subject = '[SUV-RESULT] {}'.format(card.name) - msg_body = ('New results posted for {} version {}\n\n' - '{}\n{}'.format(args.snap, args.version, card.name, card.short_url)) - try: - send_mail(to=args.email, subject=msg_subject, body=msg_body) - except Exception as e: - print('WARNING: Unable to send email: {}'.format(e)) - if not args.cardonly: - summary = '**[TESTFLINGER] {} {} {} ({}) {}**\n---\n\n'.format( - args.name, args.snap, args.version, args.revision, args.channel) - summary += '- Jenkins build details: {}\n'.format(jenkins_link) - summary += '- Full results at: {}\n\n```\n'.format(c3_link) - summary_data = args.summary.read() - summary += summary_data - summary += '\n```\n' - comment = card.comment(summary) - comment_link = "{}#comment-{}".format(card.url, comment['id']) - else: - summary_data = "" - attach_labels(board, card, snap_labels) - card_checklists = config.get(args.snap, {}).get('checklists', {}) - for checklist_name in card_checklists.keys(): - # expected_tests section in the yaml could be either a list of all - # devices we want to see tests results on, or a dict of tracks with - # those devices specified in a list under each track (ex. pi-kernel) - expected_tests = card_checklists.get( - checklist_name, {}).get('expected_tests', []) - if isinstance(expected_tests, dict): - # If multiple tracks are defined, only look at expected tests - # for the track we care about - expected_tests = expected_tests.get(track, []) - checklist = find_or_create_checklist( - card, checklist_name, expected_tests) - if args.cardonly: - item_content = "{} ({})".format(args.name, 'In progress') - else: - item_content = "[{}]({}) ({})".format( - args.name, comment_link, datetime.utcnow().isoformat()) - if jenkins_link: - item_content += " [[JENKINS]({})]".format(jenkins_link) - if c3_link: - item_content += " [[C3]({})]".format(c3_link) - elif not args.cardonly: - # If there was no c3_link, it's because the submission failed - attach_labels(board, card, ['TESTFLINGER CRASH']) - - change_checklist_item( - checklist, args.name, item_content, - checked=no_new_fails_or_skips(summary_data)) - - if not [c for c in card.fetch_checklists() if c.name == 'Sign-Off']: - checklist = find_or_create_checklist(card, 'Sign-Off') - checklist.add_checklist_item('Clear for Landing', True) - checklist.add_checklist_item('Ready for Edge', True) - checklist.add_checklist_item('Ready for Beta') - if args.channel == 'beta': - checklist.set_checklist_item('Ready for Beta', True) - checklist.add_checklist_item('Ready for Candidate') - checklist.add_checklist_item('Ready for Stable') - checklist.add_checklist_item('Can be Archived') - checklist = find_or_create_checklist(card, 'Revisions') - rev = '{} ({})'.format(args.revision, args.arch) - if rev not in [item['name'] for item in checklist.items]: - checklist.add_checklist_item(rev) - - -if __name__ == "__main__": - main() diff --git a/trello-sru-bug-manager.py b/trello-sru-bug-manager.py deleted file mode 100755 index 6f0159b..0000000 --- a/trello-sru-bug-manager.py +++ /dev/null @@ -1,383 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2018 Canonical Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# Written by: -# Paul Larson <[email protected]> - -import argparse -import datetime -import os -import re -import requests -import sys -import traceback -import json - -from launchpadlib.launchpad import Launchpad, uris -from trello import TrelloClient -from trello.exceptions import ResourceUnavailable - - -class LPHelper: - def __init__(self, credentials, project, staging): - lp_server = uris.STAGING_SERVICE_ROOT if staging else uris.LPNET_SERVICE_ROOT - self.lp = Launchpad.login_with( - sys.argv[0], lp_server, credentials_file=credentials) - self.project = self.lp.projects(project) - self.srubugs = [] - - def _build_sru_bugs(self): - """Find a bug in this project with the specified search_text""" - if not self.srubugs: - try: - # Take the kernel sru bug status data and make a list out of it - # so we can use it more easily - url = ('https://kernel.ubuntu.com/~kernel-ppa/status/' - 'swm/status.json') - bugdata = json.loads(requests.get(url).content)['trackers'] - for x in bugdata: - # Add a 'bug' field that we can use later - bugdata[x]['bug'] = x - self.srubugs.append(bugdata[x]) - except Exception: - print('ERROR: Importing bug data from {}'.format(url)) - - def find_sru_bug(self, target, version, target_type="snap"): - self._build_sru_bugs() - - if target_type == "snap": - return self.find_sru_bug_snap(target, version) - else: - return self.find_sru_bug_deb(target, version) - - def find_sru_bug_snap(self, snap, version): - try: - # Only match if there's a valid task for this snap name, and - # version is the same - for bug in self.srubugs: - if (bug.get('version') == version and - bug.get('task', {}).get( - 'snap-certification-testing', {}).get( - 'target') == snap): - return SruBug( - self.lp.bugs(bug.get('bug')), - bug.get('cycle')) - # If we get this far, no bug was found - raise LookupError - except Exception: - traceback.print_exc() - print('WARNING: something went wrong, but continuing with other cards...') - raise LookupError - - def find_sru_bug_deb(self, stack, version): - series = stack.split('-')[0] - package = 'linux' - if 'hwe' in stack: - if not 'xenial' in series: - package += '-hwe-' + version.split('.')[0] + '.' + version.split('.')[1] - else: - package += '-hwe' - # TODO: this will evolve when time goes by - elif 'oem' in stack: - series = series.split('_')[1] - package += '-oem-' + version.split('.')[0] + '.' + version.split('.')[1] - - try: - # Only match if there's a valid task for this stack name, and - # version is the same - for bug in self.srubugs: - if ( - bug.get('version') == version and - bug.get('package') == package and - bug.get('series') == series and - bug.get('variant') == 'debs' and - bug.get('task').get("kernel-sru-workflow").get('status') == 'In Progress' - ): - return SruBug( - self.lp.bugs(bug.get('bug')), - bug.get('cycle')) - # If we get this far, no bug was found - raise LookupError - except Exception: - traceback.print_exc() - print('WARNING: something went wrong, but continuing with other cards...') - raise LookupError - - -class TrelloHelper: - def __init__(self, api_key, token, board): - self.client = TrelloClient(api_key=api_key, token=token) - self.board = self.client.get_board(board) - - def _get_lane(self, lane_id): - """Search (case insensitive) for a lane called <lane_id>""" - for l in self.board.list_lists(): - lane_name = l.name.lower() - if lane_name == lane_id: - return l - raise LookupError('Trello lane "{}" not found!'.format(lane_id)) - - def search_cards_in_lane(self, lane_id, search_text): - """Yield cards with search_text in lane_id one at a time""" - lane = self._get_lane(lane_id) - for card in lane.list_cards(): - if search_text in card.name: - yield card - - -class SruBug: - """Simplify operations we care about on an LP bug""" - def __init__(self, bug, cycle): - self.bug = bug - self.id = bug.id - self.web_link = bug.web_link - self.cycle = cycle - self.calculate_due_date() - - def __repr__(self): - return self.bug.title - - def calculate_due_date(self): - try: - cycle_parts = tuple( - int(x) for x in self.cycle.split('-')[0].split('.')) - cycle_start = datetime.datetime(*cycle_parts, hour=12) - # Since cycle starts on a Monday, +11 days is the due date - # (Friday of the 2nd week in the cycle) - self.due_date = cycle_start + datetime.timedelta(days=11) - except Exception: - self.due_date = None - traceback.print_exc() - print("WARNING: Unable to set due date for {}".format(self.id)) - - def get_task_state(self, task_name): - for task in self.bug.bug_tasks: - if task_name in str(task): - return task - raise LookupError - - def set_task_state(self, task_name, state): - for task in self.bug.bug_tasks: - if task_name in str(task): - task.status = state - task.lp_save() - - def add_comment(self, comment_text): - self.bug.newMessage(content=comment_text) - - -def environ_or_required(key): - """Mapping for argparse to supply required or default from $ENV.""" - if os.environ.get(key): - return {'default': os.environ.get(key)} - else: - return {'required': True} - - -def attach_labels(board, card, label_list): - for labelstr in label_list: - for label in board.get_labels(): - if label.name == labelstr: - labels = card.labels or [] - if label not in labels: - # Avoid crash if checking labels fails to find it - try: - card.add_label(label) - except ResourceUnavailable: - pass - break - - -def get_checklist_value(checklist, key): - """Return the value of an item in a Trello checklist""" - index = checklist._get_item_index(key) - return checklist.items[index]['checked'] - - -def get_card_snap_version(card): - """Return snap name and version for a card, if formatted as expected""" - m = re.match( - r"(?P<snap>.*?)(?:\s+\-\s+)(?P<version>.*?)(?:\s+\-\s+)" - r"\((?P<revision>.*?)\)(?:\s+\-\s+\[(?P<track>.*?)\])?", card.name) - if m: - return (m.group('snap'), m.group('version')) - raise ValueError - - -def get_card_deb_version(card): - """Return snap name and version for a card, if formatted as expected""" - m = re.match( - r"(?P<stack>.*?)(?:\s+\-\s+)(?P<package>.*?)(?:\s+\-\s+)" - r"\((?P<version>.*?)\)", card.name) - if m: - return (m.group('stack'), m.group('version')) - raise ValueError - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--project', help="Launchpad project to search", - **environ_or_required('LAUNCHPAD_PROJECT')) - parser.add_argument('--credentials', help="Specify launchpad credentials", - **environ_or_required('LAUNCHPAD_CREDENTIALS')) - parser.add_argument('--key', help="Trello API key", - **environ_or_required('TRELLO_API_KEY')) - parser.add_argument('--token', help="Trello OAuth token", - **environ_or_required('TRELLO_TOKEN')) - parser.add_argument('--board', help="Trello board identifier", - **environ_or_required('TRELLO_BOARD')) - parser.add_argument('--deb', action='store_true', - help='Work on deb instead of snap') - parser.add_argument('--staging', action='store_true', - help='Use staging Launchpad for testing this script') - return parser.parse_args() - - -def card_ready_for_candidate(card): - """Return True if the signoff checkbox 'Ready for Candidate' is checked""" - # Find the Sign-Off checklist - try: - checklist = [x for x in card.fetch_checklists() - if x.name == 'Sign-Off'][0] - except IndexError: - print("WARNING: No Sign-Off checklist found!") - return False - return get_checklist_value(checklist, 'Ready for Candidate') - - -def card_ready_for_updates(card): - """Return True if the signoff checkbox 'Ready for Updates' is checked""" - # Find the Sign-Off checklist - try: - checklist = [x for x in card.fetch_checklists() - if x.name == 'Sign-Off'][0] - except IndexError: - print("WARNING: No Sign-Off checklist found!") - return False - return get_checklist_value(checklist, 'Ready for Updates') - - -def process_snaps(lp, trello): - print("Processing SRU snaps ready for promotion...") - for card in trello.search_cards_in_lane('beta', '-kernel'): - print('{} ({})'.format(card.name, card.short_url)) - # If we can't get version from title, it's not formatted how we - # expect, so ignore it - try: - snap, version = get_card_snap_version(card) - except ValueError: - continue - - try: - bug = lp.find_sru_bug(snap, version) - except LookupError: - print( - 'No bug found for {} or bug is already closed'.format(version)) - continue - - add_bug_description(bug, card) - - # Add due date to the card - if bug.due_date: - card.set_due(bug.due_date) - - # Automatically mark our task "In Progress" if it's "Confirmed" - TARGET_TASK = 'snap-certification-testing' - if bug.get_task_state(TARGET_TASK).status == 'Confirmed': - bug.set_task_state(TARGET_TASK, 'In Progress') - - if not card_ready_for_candidate(card): - continue - - update_lp(bug, 'snap-certification-testing', card) - - -def process_debs(lp, trello): - print("Processing SRU kernel debs ready for Updates repository...") - for card in trello.search_cards_in_lane('proposed', 'linux-image'): - print('{} ({})'.format(card.name, card.short_url)) - # If we can't get version from title, it's not formatted how we - # expect, so ignore it - try: - stack, version = get_card_deb_version(card) - except ValueError: - continue - - try: - bug = lp.find_sru_bug(stack, version, 'deb') - except LookupError: - print( - 'No bug found for {} or bug is already closed'.format(version)) - continue - - add_bug_description(bug, card) - - # Add due date to the card - if bug.due_date: - card.set_due(bug.due_date) - - # Automatically mark our task "In Progress" if it's still "Confirmed" - TARGET_TASK = 'certification-testing' - if bug.get_task_state(TARGET_TASK).status == 'Confirmed': - bug.set_task_state(TARGET_TASK, 'In Progress') - - if card_ready_for_updates(card): - attach_labels(trello.board, card, ['READY FOR UPDATES']) - else: - continue - - update_lp(bug, 'certification-testing', card) - - -def add_bug_description(bug, card): - desc = "[[{}]({})] - {}".format(bug.id, bug.web_link, bug) - card.set_description(desc) - - -def update_lp(bug, target_task, card): - print('- LP:{}'.format(bug.id)) - # If the bug is already fix-released, there's nothing more to do - TARGET_TASK = target_task - try: - if bug.get_task_state(TARGET_TASK).status == 'Fix Released': - print('- {} task already completed.'.format(TARGET_TASK)) - return - except LookupError: - print('ERROR: No task called "{}" found!'.format(TARGET_TASK)) - return - - # This is the bug for the card we found, mark the task complete and - # add a comment - print('- Marking {} task complete.'.format(TARGET_TASK)) - bug.set_task_state(TARGET_TASK, 'Fix Released') - comment = ("Kernel deb testing completes, no regressions found. Ready " - "for Updates. Results here: {}".format(card.url)) - bug.add_comment(comment) - - -def main(): - args = get_args() - lp = LPHelper(args.credentials, args.project, args.staging) - trello = TrelloHelper(args.key, args.token, args.board) - - if args.deb: - process_debs(lp, trello) - else: - process_snaps(lp, trello) - - -if __name__ == "__main__": - main()
-- Mailing list: https://launchpad.net/~canonical-hw-cert Post to : [email protected] Unsubscribe : https://launchpad.net/~canonical-hw-cert More help : https://help.launchpad.net/ListHelp
