Muehlenhoff has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/368190 )
Change subject: Adapt debdeploy server components to Cumin
......................................................................
Adapt debdeploy server components to Cumin
Work in progress to convert away from Salt, add separate
binary packages for further testing in labs.
Change-Id: I2a440d4725d7582f2e23747500d9b8906dbbbc0d
---
M clients/debdeploy-deploy
M clients/debdeploy-restarts
M debian/changelog
M debian/control
M debian/debdeploy-client.install
A debian/debdeploy-server.dirs
A debian/debdeploy-server.install
M master/debdeploy_updatespec.py
A server/debdeploy.py
A server/debdeploy_conf.py
A server/debdeploy_updatespec.py
11 files changed, 349 insertions(+), 6 deletions(-)
Approvals:
Muehlenhoff: Looks good to me, approved
jenkins-bot: Verified
diff --git a/clients/debdeploy-deploy b/clients/debdeploy-deploy
index 1f1f8c7..64e85e3 100755
--- a/clients/debdeploy-deploy
+++ b/clients/debdeploy-deploy
@@ -43,7 +43,7 @@
def setup_logger(verbose=False, console_output=False):
- log_file = ''
+ log_file = '/var/log/debdeploy/debdeploy.log'
log_path = os.path.dirname(log_file)
if not os.path.exists(log_path):
diff --git a/clients/debdeploy-restarts b/clients/debdeploy-restarts
index 50706e2..f36e982 100755
--- a/clients/debdeploy-restarts
+++ b/clients/debdeploy-restarts
@@ -27,7 +27,7 @@
help='Enable additional console output')
p.add_argument('--json', action='store_true', default=False,
help='Return results as JSON')
- p.add_argument('--soname', action='store', nargs='+', required=True)
+ p.add_argument('--libname', action='store', nargs='+', required=True)
args = p.parse_args(sys.argv[1:])
@@ -115,8 +115,8 @@
for i in deleted_files:
procname, pid, fname = (i)
- for soname in args.soname:
- if fname.find(soname) != -1:
+ for libname in args.libname:
+ if fname.find(libname) != -1:
if not restarts_needed.get(procname, None):
restarts_needed[procname] = {}
diff --git a/debian/changelog b/debian/changelog
index 0bfdc32..ec2e473 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -25,6 +25,7 @@
* Create a new package debdeploy-client for the Cumin-based client
which is co-installable with the Salt minion (still needed for
a transition period)
+ * Create a new package debdeploy-server for the Cumin-based server
-- Moritz Muehlenhoff <mmuhlenh...@wikimedia.org> Fri, 07 Jul 2017 13:49:03
+0200
diff --git a/debian/control b/debian/control
index 900be08..650abf5 100644
--- a/debian/control
+++ b/debian/control
@@ -28,3 +28,9 @@
Depends: lsof, python-debian, ${misc:Depends}
Description: Debdeploy central package management (client)
This is the client package of Debdeploy.
+
+Package: debdeploy-server
+Architecture: all
+Depends: lsof, python-debian, ${misc:Depends}
+Description: Debdeploy central package management (server)
+ This is the server package of Debdeploy.
diff --git a/debian/debdeploy-client.install b/debian/debdeploy-client.install
index 60b0db4..f558e9b 100644
--- a/debian/debdeploy-client.install
+++ b/debian/debdeploy-client.install
@@ -1,2 +1,3 @@
-client/debdeploy-deploy /usr/bin/
+clients/debdeploy-deploy /usr/bin/
+clients/debdeploy-restarts /usr/bin/
diff --git a/debian/debdeploy-server.dirs b/debian/debdeploy-server.dirs
new file mode 100644
index 0000000..6802d02
--- /dev/null
+++ b/debian/debdeploy-server.dirs
@@ -0,0 +1,4 @@
+/usr/lib/python2.7/dist-packages/
+/usr/sbin/
+/usr/bin/
+
diff --git a/debian/debdeploy-server.install b/debian/debdeploy-server.install
new file mode 100644
index 0000000..004608c
--- /dev/null
+++ b/debian/debdeploy-server.install
@@ -0,0 +1,3 @@
+server/debdeploy_updatespec.py /usr/lib/python2.7/dist-packages/
+server/debdeploy_conf.py /usr/lib/python2.7/dist-packages/
+server/debdeploy.py /usr/sbin
diff --git a/master/debdeploy_updatespec.py b/master/debdeploy_updatespec.py
index c683883..68da08e 100644
--- a/master/debdeploy_updatespec.py
+++ b/master/debdeploy_updatespec.py
@@ -63,7 +63,7 @@
if supported_distros.count(i) >= 1:
self.fixes[i] = updatefile["fixes"].get(i)
else:
- print "Invalid YAML file,", i, "is not a supported
distribution. You need to activate it in /deb/debdeploy.conf"
+ print "Invalid YAML file,", i, "is not a supported
distribution. You need to activate it in /etc/debdeploy.conf"
sys.exit(1)
# Local variables:
diff --git a/server/debdeploy.py b/server/debdeploy.py
new file mode 100755
index 0000000..259f096
--- /dev/null
+++ b/server/debdeploy.py
@@ -0,0 +1,206 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+# TODO:
+# reinstate rollback handling
+# revamp and readd restart handling
+
+import argparse
+import code
+import json
+import logging
+import os
+import pkgutil
+import signal
+import sys
+import datetime
+
+from ClusterShell.NodeSet import NodeSet
+
+import cumin
+
+from cumin import backends, query, transport, transports
+
+
+if os.geteuid() != 0:
+ print "debdeploy needs to be run as root"
+ sys.exit(1)
+
+from debdeploy_conf import *
+
+cumin_config = cumin.Config()
+conf = DebDeployConfig("/etc/debdeploy.conf")
+
+if conf.debug:
+ logging.basicConfig(filename='/var/log/debdeploy/debdeploy.log',
format='%(levelname)s: %(asctime)s : %(funcName)s : %(message)s',
level=logging.DEBUG)
+else:
+ logging.basicConfig(filename='/var/log/debdeploy/debdeploy.log',
format='%(levelname)s: %(asctime)s : %(funcName)s : %(message)s',
level=logging.INFO)
+
+import pydoc
+from debdeploy_updatespec import *
+
+class logpager:
+ threshold = 20 # if pager buffer contains more than <threshold> lines, use
the pager
+ def __init__(self):
+ self.buf = ""
+ def add(self, *args):
+ for i in args:
+ self.buf += str(i)
+ self.buf += "\n"
+ def add_nb(self, *args):
+ for i in args:
+ self.buf += str(i)
+ def show(self):
+ if self.buf.count("\n") > self.threshold:
+ pydoc.pager(self.buf)
+ else:
+ print self.buf
+
+
+
+def deploy_update(source, update_type, update_file, servergroup,
supported_distros, fixes):
+ '''
+ Initiate a deployment.
+
+ source : Name of the source package (string)
+ update_type : Various types of packages have different outcome, see
doc/readme.txt (string)
+ update_file : Filename of update specification (string)
+ servergroup : The name of the server group (string)
+ '''
+
+ update_desc = {}
+ update_desc["tool"] = "Non-daemon update, no service restart needed"
+ update_desc["daemon-direct"] = "Daemon update without user impact"
+ update_desc["daemon-disrupt"] = "Daemon update with service availability
impact"
+ update_desc["library"] = "Library update, several services might need to
be restarted"
+
+ print "Rolling out", source, ":",
+ print update_desc[update_type]
+
+ worker = transport.Transport.new(cumin_config, logging)
+ hosts = query.Query(cumin_config).execute('A:all')
+
+ cmd = '/usr/bin/debdeploy-deploy --source ' + source + ' --updatespec '
+
+
+ for distro in fixes:
+ if fixes[distro]:
+ cmd += supported_distros[distro][0][0] + "_" +
supported_distros[distro][0][1] + "_" + fixes[distro] + " "
+
+ worker.target = transports.Target(hosts, batch_size=100, batch_sleep=None,
logger=logging)
+ worker.commands = [ cmd ]
+
+ worker.timeout = None
+ worker.handler = 'sync'
+ worker.success_threshold = 0.1
+ worker.batch_size = 100
+ worker.batch_sleep = None
+
+ # with open('/dev/null', 'w') as discard_output:
+ # oldstdout = sys.stdout
+ # sys.stdout = discard_output
+ # exit_code = worker.execute()
+ # sys.stdout = oldstdout
+
+ exit_code = worker.execute()
+
+ out = {}
+ for nodeset, output in worker.get_results():
+ print output
+
+# print worker.handler.counters
+
+
+def detect_restarts(libnames, servergroup):
+ '''
+ Query for necessary restarts after a library or interpreter upgrade
+
+ libnames : A list of library base names, e.g. libssl for
/usr/lib/x86_64-linux-gnu/libssl.so.1.0.0 (list of strings)
+ servergroup : The name of the server group for which process restarts
should be queried (string)
+ '''
+
+ worker = transport.Transport.new(cumin_config, logging)
+ hosts = query.Query(cumin_config).execute('A:all')
+
+ cmd = '/usr/bin/debdeploy-restarts --libname '
+ for lib in libnames:
+ cmd += lib + " "
+
+ worker.target = transports.Target(hosts, batch_size=100, batch_sleep=None,
logger=logging)
+ worker.commands = [ cmd ]
+
+ worker.timeout = None
+ worker.handler = 'sync'
+ worker.success_threshold = 0.1
+ worker.batch_size = 100
+ worker.batch_sleep = None
+
+ # with open('/dev/null', 'w') as discard_output:
+ # oldstdout = sys.stdout
+ # sys.stdout = discard_output
+ # exit_code = worker.execute()
+ # sys.stdout = oldstdout
+
+ exit_code = worker.execute()
+
+ out = {}
+ for nodeset, output in worker.get_results():
+ print output
+
+
+def main():
+ p = argparse.ArgumentParser(usage="debdeploy-master [options] command
<cmd-option>\n \
+ The following commands are supported: \n\n \
+ deploy : Install a software update, requires --update
and --servers \n \
+ query_restart : Query for necessary restarts after a library
or interpreter upgrade \n \
+ rollback : Rollback a software deployment")
+
+ p.add_argument("-u", "--update", action="store", type=str,
dest="updatefile", help="A YAML file containing the update specification (which
source package to update and the respective fixed versions")
+ p.add_argument("-s", "--servers", action="store", type=str,
dest="serverlist", help="The group of servers on which the update should be
applied")
+ p.add_argument("--verbose", action="store_true", dest="verbose",
help="Enable verbose output, e.g. show full apt output in status-deploy and
status-rollback")
+
+ p.add_argument("command")
+ p.add_argument("command_option", nargs="?", default="unset")
+
+ opt = p.parse_args()
+
+ if opt.command in ("deploy", "rollback", "restart", "query_restart"):
+ if not opt.serverlist:
+ p.error("You need to provide a server list (-s)")
+
+ if opt.command in ("deploy", "rollback", "query_restart"):
+ if not opt.updatefile:
+ p.error("You need to provide an update file (-u)")
+
+ if opt.command in ("restart"):
+ if not opt.program:
+ p.error("You need to provide a program to restart (-p)")
+
+ if opt.command == "deploy":
+ update = DebDeployUpdateSpec(opt.updatefile, conf.supported_distros)
+ deploy_update(update.source, update.update_type, opt.updatefile,
opt.serverlist, conf.supported_distros, update.fixes)
+
+ elif opt.command == "query_restart":
+ update = DebDeployUpdateSpec(opt.updatefile, conf.supported_distros)
+ detect_restarts(update.libraries, opt.serverlist)
+
+ elif opt.command == "status-rollback":
+ display_status(rollback_mode=True)
+
+ elif opt.command == "rollback":
+ rollback(opt.serverlist, opt.updatefile)
+
+
+if __name__ == '__main__':
+ print main()
+
+
+# Local variables:
+# mode: python
+# End:
+
+
+
+
+
+
diff --git a/server/debdeploy_conf.py b/server/debdeploy_conf.py
new file mode 100644
index 0000000..c721bf3
--- /dev/null
+++ b/server/debdeploy_conf.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+import ConfigParser, sys
+
+class DebDeployConfig(object):
+ '''
+ Class to read/provide the system-wide configuration of the debdeploy
master component.
+ It contains the following variables:
+
+ supported_distros: List of strings of supported distros (per Debian/Ubuntu
codename)
+ '''
+ supported_distros = {}
+ debug = False
+
+ def __init__(self, configfile):
+ config = ConfigParser.ConfigParser()
+ if len(config.read(configfile)) == 0:
+ print "/etc/debdeploy.conf doesn't exist, you need to create it."
+ print "See /usr/share/doc/debdeploy-master/examples/debdeploy.conf"
+ sys.exit(1)
+
+ if not config.has_section("distros"):
+ print "Could not read list of supported distributions, make sure",
configfile, "contains a section [distros]"
+ sys.exit(1)
+
+ for distro in config.options("distros"):
+ self.supported_distros[distro] = []
+ self.supported_distros[distro].append([x.strip() for x in
config.get("distros", distro).split(",")])
+
+ if len(self.supported_distros) < 1:
+ print "You need to specify at least one supported distribution in
/etc/debdeploy.conf"
+ sys.exit(1)
+
+ if config.has_section("logging") and config.has_option("logging",
"debug"):
+ if config.getboolean("logging", "debug"):
+ self.debug = True
+
+conf = DebDeployConfig("/etc/debdeploy.conf")
+
+# Local variables:
+# mode: python
+# End:
+
+
+
+
+
diff --git a/server/debdeploy_updatespec.py b/server/debdeploy_updatespec.py
new file mode 100644
index 0000000..29e2d8a
--- /dev/null
+++ b/server/debdeploy_updatespec.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+
+import salt.client
+import yaml
+import sys
+
+class DebDeployUpdateSpec(object):
+ '''
+ Each update is described in a YAML file, see docs/readme.txt for the data
+ format.
+ '''
+
+ source = ""
+ comment = ""
+ update_type = ""
+ fixes = {}
+ libraries = []
+ legit_type = ['tool', 'daemon-direct', 'daemon-disrupt', 'daemon-cluster',
'reboot', 'reboot-cluster', 'library']
+
+ def __init__(self, updatespec, supported_distros):
+ '''
+ Parse an update spec file.
+
+ updatespec : Filename of the update spec file (string)
+ supported_distros : These are the distro codenames for which a fixed
version can be provided (list of strings)
+ '''
+
+ try:
+ with open(updatespec, "r") as stream:
+ updatefile = yaml.load(stream)
+
+ except IOError:
+ print "Error: Could not open", updatespec
+ sys.exit(1)
+
+ except yaml.scanner.ScannerError, e:
+ print "Invalid YAML file:"
+ print e
+ sys.exit(1)
+
+ if not updatefile.has_key("source"):
+ print "Invalid YAML file, you need to specify the source package
using the 'source' stanza, see the annotated example file for details"
+ sys.exit(1)
+ else:
+ self.source = updatefile["source"]
+
+ if not updatefile.has_key("update_type"):
+ print "Invalid YAML file, you need to specify the type of update
using the 'update_type' stanza, see the annotated example file for details"
+ sys.exit(1)
+ else:
+ if updatefile["update_type"] not in self.legit_type:
+ print "Invalid YAML file, invalid 'update_type'"
+ sys.exit(1)
+ self.update_type = updatefile["update_type"]
+
+ if updatefile.has_key("comment"):
+ self.comment = updatefile["comment"]
+
+ if updatefile.has_key("libraries"):
+ self.libraries = updatefile["libraries"]
+
+ if not updatefile.has_key("fixes"):
+ print "Invalid YAML file, you need to specify at least one fixed
version using the 'fixes' stanza, see the annotated example file for details"
+ sys.exit(1)
+ else:
+ for i in updatefile["fixes"]:
+ if len(supported_distros.keys()) >= 1:
+ self.fixes[i] = updatefile["fixes"].get(i)
+ else:
+ print "Invalid YAML file,", i, "is not a supported
distribution. You need to activate it in /deb/debdeploy.conf"
+ sys.exit(1)
+
+# Local variables:
+# mode: python
+# End:
--
To view, visit https://gerrit.wikimedia.org/r/368190
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I2a440d4725d7582f2e23747500d9b8906dbbbc0d
Gerrit-PatchSet: 6
Gerrit-Project: operations/debs/debdeploy
Gerrit-Branch: master
Gerrit-Owner: Muehlenhoff <mmuhlenh...@wikimedia.org>
Gerrit-Reviewer: Muehlenhoff <mmuhlenh...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
