Volans has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/394990 )
Change subject: Add CLI script to be installed in the target hosts ...................................................................... Add CLI script to be installed in the target hosts Bug: T167504 Change-Id: I3b3cab366e24f00f3dce4edadbe15d3c83a02813 --- A utils/cli.py 1 file changed, 291 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/software/debmonitor refs/changes/90/394990/1 diff --git a/utils/cli.py b/utils/cli.py new file mode 100644 index 0000000..41604cd --- /dev/null +++ b/utils/cli.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python +# ---------------------------------------------------------------------------- +# DebMonitor CLI - Debian packages tracker CLI +# Copyright (C) 2017 Riccardo Coccioli <[email protected]> +# Wikimedia Foundation, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# ---------------------------------------------------------------------------- +""" +DebMonitor CLI - Debian packages tracker CLI. + +Automatically collect the current status of all installed and upgradable packages and report it to a DebMonitor server. + +This script was tested with Python 2.7,3.5,3.6. + +* Install the following Debian packages dependencies, choosing either the Python2 or the Python3 variant based on which + version of Python will be used to run this script: + + * python-apt + * python-requests + +* Deploy this standole CLI script across the fleet, for example into ``/usr/local/bin/debmonitor``, and make it + executable. +* Add a configuration file in ``/etc/apt/apt.conf.d/`` with the following content, replacing ``##DEMBONITOR_SERVER##`` + with the domain name at which the DebMonitor server is reachable. + + .. code-block:: none + + # Tell dpkg to use version 3 of the protocol for the Pre-Install-Pkgs hook (version 2 is also supported) + Dpkg::Tools::options::/usr/local/bin/debmonitor::Version "3"; + # Set the dpkg hook to call DebMonitor for any change with the -g/--dpkg option to read the changes from stdin + Dpkg::Pre-Install-Pkgs {"/usr/local/bin/debmonitor -s ##DEMBONITOR_SERVER## -g || true";}; + # Set the APT update hook to call DebMonitor with the -u/upgradables option to send only the pending upgrades + APT::Update::Post-Invoke {"/usr/local/bin/debmonitor -s ##DEMBONITOR_SERVER## -u || true"}; + +* Set a daily or weekly crontab that executes DebMonitor to send the list of all installed and upgradable packages + (do not set the ``-g`` or ``-u`` options). It is used as a reconciliation method if any of the hook would fail. + It is also required to run DebMonitor in full mode at least once to track all the packages. + +""" +from __future__ import print_function + +import argparse +import json +import logging +import platform +import socket +import sys + +from collections import namedtuple + +import apt +import requests + + +SECURITY_UPGRADE = 'security' +logger = logging.getLogger('debmonitor') +AptLineV2 = namedtuple('LineV2', ['name', 'version_from', 'direction', 'version_to', 'action']) +AptLineV3 = namedtuple('LineV3', ['name', 'version_from', 'arch_from', 'multiarch_from', 'direction', 'version_to', + 'arch_to', 'multiarch_to', 'action']) + + +class AptInstalledFilter(apt.cache.Filter): + """Filter for python-apt to filter only installed packages.""" + + def apply(self, pkg): + """Filter only installed packages.""" + if pkg.is_installed: + return True + + return False + + +def get_packages(upgradables_only=False): + """Return the list of installed and upgradable packages.""" + packages = {'installed': [], 'upgradables': [], 'uninstalled': []} + cache = apt.cache.FilteredCache() + cache.set_filter(AptInstalledFilter()) + logger.info('Found %d installed binary packages', len(cache)) + + cache.upgrade(dist_upgrade=True) + upgrades = cache.get_changes() + logger.info('Found %d upgradable binary packages (including new dependencies)', len(upgrades)) + + if not upgradables_only: + for pkg in cache: + package = {'name': pkg.name, 'version': pkg.installed.version, 'source': pkg.installed.source_name} + packages['installed'].append(package) + logger.debug('Collected installed: %s', package) + + for pkg in upgrades: + if not pkg.is_installed: + continue + + upgrade = {'name': pkg.name, 'version_from': pkg.installed.version, 'version_to': pkg.candidate.version, + 'source': pkg.candidate.source_name, 'type': get_upgrade_type(pkg.candidate)} + packages['upgradables'].append(upgrade) + logger.debug('Collected upgrade: %s', upgrade) + + return packages + + +def get_upgrade_type(candidate): + """Return the upgrade type of the candidate package version.""" + for origin in candidate.origins: + if origin.origin == 'Debian' and (origin.label == 'Debian-Security' or origin.site == 'security.debian.org'): + return SECURITY_UPGRADE + + if origin.origin == 'Ubuntu' and origin.archive.endswith('-security'): + return SECURITY_UPGRADE + + return '' + + +def parse_packages(input_lines): + """Parse packages changes as reported by the Dpkg::Pre-Install-Pkgs hook.""" + version_line = input_lines.pop(0).strip() + + if not version_line.startswith('VERSION '): + raise RuntimeError('Expected VERSION line to be the first one, got: {version}'.format(version=version_line)) + + version = int(version_line[-1]) + upgrades = input_lines[input_lines.index('\n') + 1:] + if not upgrades: + return {} + + packages = {'installed': [], 'upgradables': [], 'uninstalled': []} + cache = apt.cache.Cache() + for update_line in upgrades: + group, package = parse_apt_line(update_line, cache, version=version) + if group is not None: + packages[group].append(package) + + logger.info('Got %d updates from dpkg hook version %d', len(packages['installed']) + len(packages['uninstalled']), + version) + return packages + + +def parse_apt_line(update_line, cache, version=3): + """Parse a single package line as reported by the Dpkg::Pre-Install-Pkgs hook version 3 or 2. + + Example version 2 upgrade: + package-name 1.0.0-1 < 1.0.0-2 /var/cache/apt/archives/package-name_1.0.0-1_all.deb + package-name 1.0.0-1 < 1.0.0-2 **CONFIGURE** + + Example version 3 upgrade: + package-name 1.0.0-1 all none < 1.0.0-2 all none /var/cache/apt/archives/package-name_1.0.0-1_all.deb + package-name 1.0.0-1 all none < 1.0.0-2 all none **CONFIGURE** + + """ + if version == 3: + line = AptLineV3(*update_line.strip().split(' ', 9)) + elif version == 2: + line = AptLineV2(*update_line.strip().split(' ', 5)) + else: + raise RuntimeError('Unsupported version {ver}'.format(ver=version)) + + if line.action in ('**CONFIGURE**', '**REMOVE**'): # Skip those lines, package already tracked + return None, None + + cache_item = cache[line.name] + if line.direction == '<': # Upgrade + group = 'installed' + package = {'name': line.name, 'version': line.version_to, 'source': cache_item.candidate.source_name} + logger.debug('Collected upgraded package: %s', package) + + elif line.description == '>': # Downgrade/removal + if line.version_to == '-': # Removal + group = 'uninstalled' + package = {'name': line.name, 'version': line.version_from, 'source': cache_item.installed.source_name} + logger.debug('Collected removed package: %s', package) + else: # Downgrade + group = 'installed' + package = {'name': line.name, 'version': line.version_to, 'source': cache_item.candidate.source_name} + logger.debug('Collected downgraded package: %s', package) + + else: # No change (=) + group = None + package = None + + return group, package + + +def parse_args(args): + """Parse command line arguments.""" + parser = argparse.ArgumentParser(prog='debmonitor', description='DebMonitor CLI - Debian packages tracker CLI', + epilog=__doc__) + parser.add_argument('-s', '--server', help='DebMonitor server DNS name, required unless -n/--dry-run is set.') + parser.add_argument('-p', '--port', default=443, type=int, + help='Port in which the DebMonitor server is listening. [default: 443]') + parser.add_argument('-c', '--cert', + help=('Path to the client SSL certificate to use for sending the update. If it does not ' + 'contain also the private key, -k/--key must be specified too.')) + parser.add_argument('-k', '--key', + help=('Path to the client SSH private key to use for sending the update. If not specified, ' + 'the private key is expected to be found in the certificate defined by -c/--cert.')) + parser.add_argument('-a', '--api', help='Version of the API to use', default='v1') + parser.add_argument('-u', '--upgradables', action='store_true', + help='Send only the list of upgradable packages. Can be used as a hook for apt-get update.') + parser.add_argument('-g', '--dpkg-hook', action='store_true', + help=('Parse modified packages from stdin according to DPKG hook Dpkg::Pre-Install-Pkgs ' + 'format for version 3 and 2.')) + parser.add_argument('-n', '--dry-run', action='store_true', + help='Do not send the report to DebMonitor server and print it to stdout.') + parser.add_argument('-d', '--debug', action="store_true", help='Set logging level to DEBUG') + parsed_args = parser.parse_args(args) + + if not parsed_args.server and not parsed_args.dry_run: + parser.error('argument -s/--server is required unless -n/--dry-run is set') + + if parsed_args.key is not None and parsed_args.cert is None: + parser.error('argument -c/--cert is required when -k/--key is set') + + if parsed_args.upgradables and parsed_args.dpkg_hook: + parser.error('argument -u/--upgradables and -g/--dpkg-hook are mutually exclusive') + + return parsed_args + + +def main(args): + """Execute the script.""" + args = parse_args(args) + + level = logging.INFO + if args.debug: + level = logging.DEBUG + logging.basicConfig(level=level) + + hostname = socket.getfqdn() + + if args.upgradables or args.dpkg_hook: + upgrade_type = 'partial' + else: + upgrade_type = 'full' + + if args.dpkg_hook: + packages = parse_packages(sys.stdin.readlines()) + else: + packages = get_packages(upgradables_only=args.upgradables) + + if not packages: + return 0 + + payload = { + 'api_version': args.api, + 'os': platform.linux_distribution()[0].title(), + 'hostname': hostname, + 'running_kernel': { + 'release': platform.release(), + 'version': platform.version(), + }, + 'installed': packages['installed'], + 'uninstalled': packages['uninstalled'], + 'upgradables': packages['upgradables'], + 'update_type': upgrade_type, + } + + if args.dry_run: + print(json.dumps(payload, sort_keys=True, indent=4)) + return 0 + + url = 'http://{server}:{port}/hosts/{host}/update'.format(server=args.server, port=args.port, host=hostname) + + cert = None + if args.key is not None: + cert = (args.cert, args.key) + elif args.cert is not None: + cert = args.cert + + response = requests.post(url, cert=cert, json=payload) + if response.status_code != requests.status_codes.codes.created: + raise RuntimeError('Failed to send the update to the DebMonitor server: {status} {body}'.format( + status=response.status_code, body=response.text)) + logger.info('Successfully sent the %s update to the DebMonitor server', upgrade_type) + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) -- To view, visit https://gerrit.wikimedia.org/r/394990 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3b3cab366e24f00f3dce4edadbe15d3c83a02813 Gerrit-PatchSet: 1 Gerrit-Project: operations/software/debmonitor Gerrit-Branch: master Gerrit-Owner: Volans <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
