Muehlenhoff has uploaded a new change for review. https://gerrit.wikimedia.org/r/242140
Change subject: Various bugfixes / tweaks ...................................................................... Various bugfixes / tweaks Move generation of spec files to a separate command installed in /usr/bin Fix syntax of generated YAML files Fix parsing of dpkg status file for binNMUed packages Change-Id: Id0ae40db09abd6c926d31a58beb03bf0f0604a30 --- M debian/changelog M debian/debdeploy-master.dirs M debian/debdeploy-master.install M docs/TODO M docs/readme.txt M master/debdeploy A master/generate-debdeploy-spec M minion/debdeploy-minion.py 8 files changed, 116 insertions(+), 91 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/debs/debdeploy refs/changes/40/242140/1 diff --git a/debian/changelog b/debian/changelog index 5a89dd7..13e09c6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +debdeploy (0.0.8-1) jessie-wikimedia; urgency=medium + + * Move generation of spec files to a separate command installed in /usr/bin + * Fix syntax of generated YAML files + * Fix parsing of dpkg status file for binNMUed packages + + -- Moritz Muehlenhoff <mmuhlenh...@wikimedia.org> Fri, 25 Sep 2015 16:52:23 +0200 + debdeploy (0.0.7-1) jessie-wikimedia; urgency=medium * Improve error handling in case of malformed YAML specs diff --git a/debian/debdeploy-master.dirs b/debian/debdeploy-master.dirs index 7c53a23..cae4528 100644 --- a/debian/debdeploy-master.dirs +++ b/debian/debdeploy-master.dirs @@ -3,4 +3,5 @@ /usr/lib/debdeploy/ /usr/lib/python2.7/dist-packages/ /usr/sbin/ +/usr/bin/ diff --git a/debian/debdeploy-master.install b/debian/debdeploy-master.install index 064b37e..1783969 100644 --- a/debian/debdeploy-master.install +++ b/debian/debdeploy-master.install @@ -2,3 +2,4 @@ master/debdeploy_updatespec.py /usr/lib/python2.7/dist-packages/ master/debdeploy_conf.py /usr/lib/python2.7/dist-packages/ master/debdeploy /usr/sbin +master/generate-debdeploy-spec /usr/bin diff --git a/docs/TODO b/docs/TODO index 95272f6..12c0381 100644 --- a/docs/TODO +++ b/docs/TODO @@ -4,7 +4,6 @@ Refinements to existing functionality: - Bash completion for commands and options -- Add a command to generate a YAML file interactively - Write initiated deployment jobs into a logfile - Provide logrotate configs @@ -14,3 +13,5 @@ and create a WMF-specific IRC notification which updates "server admin log" - Make the logging configurable and add a WMF-specific logging method for Logstash +- Implement timed restarts, which restart services not an once, but in + steps/with delays diff --git a/docs/readme.txt b/docs/readme.txt index 6add055..5ce9269 100644 --- a/docs/readme.txt +++ b/docs/readme.txt @@ -2,10 +2,10 @@ Summary ------- -debdeploy is based on salt to reuse it's authentication and transfer -capabilities. Updates are initiated via the debdeploy tool running -on the salt master. Servers can be grouped into arbitrary sets of -servers/services based on Salt grains. +debdeploy allows the deployment of software updates in Debian (or Debian-based) +environments on a large scale. It is based on salt; updates are +initiated via the debdeploy tool running on the salt master. Servers +can be grouped into arbitrary sets of servers/services based on Salt grains. Basic setup ----------- diff --git a/master/debdeploy b/master/debdeploy index dc8e05f..72cd06c 100755 --- a/master/debdeploy +++ b/master/debdeploy @@ -342,86 +342,6 @@ for i in sorted(members): print i -def generate_spec(supported_distros): - ''' - Interactivey generate an update spec file in the currently directory. - ''' - - source = "" - comment = "" - updatetype = "" - fixes = {} - - while source == "": - source = raw_input("Please enter the name of source package (e.g. openssl). Leave blank or type 'quit' to abort\n>").strip() - if source == "" or source == "quit": - print "Aborting" - sys.exit(1) - - comment = raw_input('You can enter an optional comment, e.g. a reference to a security advisory or a CVE ID mapping\n>').strip() - if comment == "quit": - print "Aborting" - sys.exit(1) - - while updatetype not in ['tool', 'daemon-direct', 'daemon-disrupt', 'reboot', 'library', 'quit']: - print "tool -> The updated packages is an enduser tool, can be" - print " rolled-out immediately." - print "daemon-direct -> Daemons which are restarted during update, but which" - print " do no affect existing users." - print "daemon-disrupt -> Daemons which are restarted during update, where the" - print " users notice an impact. The update procedure is almost" - print " identical, but displays additional warnings" - print "reboot -> Lowlevel component which requires a system reboot" - print " kernel/glibc/dbus). After installing the update, the" - print " system reboot(s) can be managed subsequently" - print "library -> After a library is updated, programs may need to be" - print " restarted to fully effect the change. In addition" - print " to libs, some applications may also fall under this rule," - print " e.g. when updating QEMU, you might need to restart VMs." - updatetype = raw_input("Please enter the update type:\n>").strip() - if source == "" or source == "quit": - print "Aborting" - sys.exit(1) - - for i in supported_distros: - fixes[i] = raw_input("Please enter the version which fixed in in " + i + ". Leave blank if no fix is available/required for a given distro.\n>").strip() - - at_least_one_fixed_version = False - for i in fixes.values(): - if i != "": - at_least_one_fixed_version = True - - if not at_least_one_fixed_version: - print "At least one fixed version needs to be configured, aborting" - sys.exit(1) - - - valid_name = False - suggested_name = datetime.datetime.now().strftime("%Y-%m-%d-") + source + ".yaml" - while not valid_name: - print "Please enter a name under which the YAML file should be created" - print "Leave blank to use ", suggested_name - yamlfilename = raw_input('>').strip() - if not yamlfilename: - yamlfilename = suggested_name - if os.path.exists(yamlfilename): - print "File name already exists, please re-enter." - else: - valid_name = True - - try: - with open(yamlfilename, "w") as yamlfile: - yamltext = 'source: ' + source + '\n' \ - 'comment: ' + comment + '\n' \ - 'update_type: ' + updatetype + '\n' \ - 'fixes:\n' - yamlfile.write(yamltext) - for i in fixes: - yamlfile.write("\t" + i + ": " + fixes[i] + "\n") - except IOError, e: - print "Error:", e - sys.exit(1) - def rollback(grains, update_file): ''' Trigger process restarts for a set of servers @@ -465,13 +385,13 @@ The following commands are supported: \n\n \ deploy : Install a software update, requires --update and --servers \n \ status-deploy : Query the status of a software deployment, requires --update and --servers\n \ + restart : Restart a service on all servers of a server group \n \ rollback : Rollback a software deployment\n \ status-rollback : Query the status of a software deployment rollback\n \ list-server-groups : Display a list of all defined server groups. If an update file is listed \n \ in addition, it shows whether an update has been applied for that group. \n \ list-server-group-members : Show all servers represented by a server group \n \ check-missing : Display a list of all servers which don't have an update installed \n \ - generate-spec : Generate an update spec file interactively \n \ pkgdb-source : Re-read the package status of a given host into the package database") @@ -510,9 +430,6 @@ elif command == "status-deploy": display_status() - -elif command == "generate-spec": - generate_spec(conf.supported_distros) elif command == "status-rollback": display_status(rollback_mode=True) diff --git a/master/generate-debdeploy-spec b/master/generate-debdeploy-spec new file mode 100755 index 0000000..e18dd33 --- /dev/null +++ b/master/generate-debdeploy-spec @@ -0,0 +1,92 @@ +#! /usr/bin/python +# -*- coding: utf-8 -*- + +import sys, optparse, logging, os, datetime +from debdeploy_conf import * + +conf = DebDeployConfig("/etc/debdeploy.conf") + +source = "" +comment = "" +updatetype = "" +fixes = {} + +while source == "": + source = raw_input("Please enter the name of source package (e.g. openssl). Leave blank or type 'quit' to abort\n>").strip() + if source == "" or source == "quit": + print "Aborting" + sys.exit(1) + +comment = raw_input('You can enter an optional comment, e.g. a reference to a security advisory or a CVE ID mapping\n>').strip() +if comment == "quit": + print "Aborting" + sys.exit(1) + +while updatetype not in ['tool', 'daemon-direct', 'daemon-disrupt', 'reboot', 'library', 'quit']: + print "tool -> The updated packages is an enduser tool, can be" + print " rolled-out immediately." + print "daemon-direct -> Daemons which are restarted during update, but which" + print " do no affect existing users." + print "daemon-disrupt -> Daemons which are restarted during update, where the" + print " users notice an impact. The update procedure is almost" + print " identical, but displays additional warnings" + print "reboot -> Lowlevel component which requires a system reboot" + print " kernel/glibc/dbus). After installing the update, the" + print " system reboot(s) can be managed subsequently" + print "library -> After a library is updated, programs may need to be" + print " restarted to fully effect the change. In addition" + print " to libs, some applications may also fall under this rule," + print " e.g. when updating QEMU, you might need to restart VMs." + updatetype = raw_input("Please enter the update type:\n>").strip() + if source == "" or source == "quit": + print "Aborting" + sys.exit(1) + +for i in conf.supported_distros: + fixes[i] = raw_input("Please enter the version which fixed in in " + i + ". Leave blank if no fix is available/required for a given distro.\n>").strip() + +at_least_one_fixed_version = False +for i in fixes.values(): + if i != "": + at_least_one_fixed_version = True + +if not at_least_one_fixed_version: + print "At least one fixed version needs to be configured, aborting" + sys.exit(1) + + +valid_name = False +suggested_name = datetime.datetime.now().strftime("%Y-%m-%d-") + source + ".yaml" +while not valid_name: + print "Please enter a name under which the YAML file should be created" + print "Leave blank to use ", suggested_name + yamlfilename = raw_input('>').strip() + if not yamlfilename: + yamlfilename = suggested_name + if os.path.exists(yamlfilename): + print "File name already exists, please re-enter." + else: + valid_name = True + +try: + with open(yamlfilename, "w") as yamlfile: + yamltext = 'source: ' + source + '\n' \ + 'comment: ' + comment + '\n' \ + 'update_type: ' + updatetype + '\n' \ + 'fixes:\n' + yamlfile.write(yamltext) + for i in fixes: + yamlfile.write(" " + i + ": " + fixes[i] + "\n") +except IOError, e: + print "Error:", e + sys.exit(1) + + +# Local variables: +# mode: python +# End: + + + + + diff --git a/minion/debdeploy-minion.py b/minion/debdeploy-minion.py index a3fbc8b..318b5f4 100644 --- a/minion/debdeploy-minion.py +++ b/minion/debdeploy-minion.py @@ -3,7 +3,7 @@ Module for deploying DEB packages on wide scale ''' -import logging, pickle, subprocess, os +import logging, pickle, subprocess, os, re import logging.handlers import salt.utils @@ -210,7 +210,12 @@ # binary packages not having the same name as the binary package installed_binary_packages = [] for pkg in deb822.Packages.iter_paragraphs(file('/var/lib/dpkg/status')): - if pkg.has_key('Source') and pkg['Source'] == source: + + # Source packages which have had a binNMU have a Source: entry with the source + # package version in brackets, so strip these + # If no Source: entry is present in /var/lib/dpkg/status, then the source package + # name is identical to the binary package name + if pkg.has_key('Source') and re.sub(r'\(.*?\)', '', pkg['Source']).strip() == source: installed_binary_packages.append({pkg['Package'] : versions[installed_distro]}) elif pkg.has_key('Package') and pkg['Package'] == source: installed_binary_packages.append({pkg['Package'] : versions[installed_distro]}) -- To view, visit https://gerrit.wikimedia.org/r/242140 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Id0ae40db09abd6c926d31a58beb03bf0f0604a30 Gerrit-PatchSet: 1 Gerrit-Project: operations/debs/debdeploy Gerrit-Branch: master Gerrit-Owner: Muehlenhoff <mmuhlenh...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits