Tim Landscheidt has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/328030 )
Change subject: Tools: Generate node sets dynamically
......................................................................
Tools: Generate node sets dynamically
Currently, the list of nodes per hostgroup is regenerated every
minute. This is far more often than clush is actually used according
to the log file, and additionally this two-step process can result in
confusing behaviour in cases where the cron job fails in some way.
This change generates the node sets dynamically, on demand. Thus they
are always up-to-date and errors in generating the list make
themselves known immediately. Technically, the generated "load" on
wikitech will be reduced, but given the low frequency of API calls
(once per clush invocation) this is not a factor.
Change-Id: I5bb630836beb1c6a2aa2b346bef03687db9f4732
---
D modules/role/files/toollabs/clush/tools-clush-generator
A modules/role/files/toollabs/clush/tools-clush-generator.py
D modules/role/files/toollabs/clush/tools-clush-interpreter
M modules/role/manifests/toollabs/clush/master.pp
4 files changed, 108 insertions(+), 131 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/30/328030/1
diff --git a/modules/role/files/toollabs/clush/tools-clush-generator
b/modules/role/files/toollabs/clush/tools-clush-generator
deleted file mode 100644
index 2b8f7cb..0000000
--- a/modules/role/files/toollabs/clush/tools-clush-generator
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/python3
-"""
-Simple script that generates a YAML file classifying all instances
-on the tools project into groups based on the role they perform.
-
-This YAML file can then be read by `tools-clush-interpreter` to
-list instances within a group. This can be used by `clush` to
-allow arbitrary command execution on targeted list of instances.
-
-This is run in a cron every hour.
-"""
-import json
-import yaml
-import argparse
-from urllib.request import urlopen
-
-
-# Maps prefixes to hostgroup names
-TOOLS_PREFIX_CLASSIFIER = {
- 'webgrid-lighttpd-12': 'webgrid-lighttpd-precise',
- 'webgrid-lighttpd-14': 'webgrid-lighttpd-trusty',
- 'webgrid-generic': 'webgrid-generic',
- 'webgrid-': 'webgrid',
- 'exec-': 'exec',
- 'exec-12': 'exec-precise',
- 'exec-14': 'exec-trusty',
- 'proxy-': 'webproxy',
- 'checker-': 'checker',
- 'redis-': 'redis',
- 'services-': 'services',
- 'bastion-': 'bastion',
- 'cron-': 'cron',
- 'grid-master': 'grid-master',
- 'grid-shadow': 'grid-shadow',
- 'mail': 'mail',
- 'static-': 'static',
- 'worker': 'k8s-worker',
- 'k8s-master': 'k8s-master',
- 'flannel-etcd': 'flannel-etcd',
- 'k8s-etcd': 'k8s-etcd',
- 'logs': 'logs',
- 'precise-dev': 'precise-dev',
- 'docker-builder': 'docker-builder',
- 'prometheus': 'prometheus',
- '': 'all',
-}
-
-
-def get_hostgroups(classifier, project_name):
- hostgroups = {name: [] for name in classifier.values()}
-
- api_url = 'https://wikitech.wikimedia.org/w/api.php' \
- '?action=query&list=novainstances&niregion=eqiad&format=json' \
- '&niproject=' + project_name
-
- data = json.loads(urlopen(api_url).read().decode('utf-8'))
-
- for instance in data['query']['novainstances']:
- name = instance['name']
- for prefix in classifier:
- if name.startswith('%s-%s' % (project_name, prefix)):
- role = classifier[prefix]
- hostgroups[role].append('%s.%s.eqiad.wmflabs' % (name,
project_name))
-
- return hostgroups
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser()
- parser.add_argument(
- 'outputpath',
- help='Path to output hostgroup to host mappings'
- )
- args = parser.parse_args()
-
- with open('/etc/wmflabs-project', 'r') as f:
- project_name = f.read().rstrip('\n')
-
- hostgroups = get_hostgroups(TOOLS_PREFIX_CLASSIFIER, project_name)
- with open(args.outputpath, 'w') as f:
- f.write(yaml.safe_dump(hostgroups, default_flow_style=False))
diff --git a/modules/role/files/toollabs/clush/tools-clush-generator.py
b/modules/role/files/toollabs/clush/tools-clush-generator.py
new file mode 100644
index 0000000..15b54c4
--- /dev/null
+++ b/modules/role/files/toollabs/clush/tools-clush-generator.py
@@ -0,0 +1,95 @@
+#!/usr/bin/python3
+
+
+"""Simple generator script for clustershell to dynamically list all
+ instances and classify them into groups based on their hostname.
+
+"""
+
+
+import argparse
+import json
+import urllib.request
+
+
+# Maps hostgroup names to prefixes.
+TOOLS_HOSTGROUPS = {
+ 'all': '',
+ 'bastion': 'bastion-',
+ 'checker': 'checker-',
+ 'cron': 'cron-',
+ 'docker-builder': 'docker-builder',
+ 'exec': 'exec-',
+ 'exec-precise': 'exec-12',
+ 'exec-trusty': 'exec-14',
+ 'flannel-etcd': 'flannel-etcd',
+ 'grid-master': 'grid-master',
+ 'grid-shadow': 'grid-shadow',
+ 'k8s-etcd': 'k8s-etcd',
+ 'k8s-master': 'k8s-master',
+ 'k8s-worker': 'worker',
+ 'logs': 'logs',
+ 'mail': 'mail',
+ 'precise-dev': 'precise-dev',
+ 'prometheus': 'prometheus',
+ 'redis': 'redis-',
+ 'services': 'services-',
+ 'static': 'static-',
+ 'webgrid': 'webgrid-',
+ 'webgrid-generic': 'webgrid-generic',
+ 'webgrid-lighttpd-precise': 'webgrid-lighttpd-12',
+ 'webgrid-lighttpd-trusty': 'webgrid-lighttpd-14',
+ 'webproxy': 'proxy-',
+}
+
+
+def list_hostgroups():
+ """List all hostgroups on stdout."""
+ for hostgroup in sorted(TOOLS_HOSTGROUPS):
+ print(hostgroup)
+
+
+def list_nodes(project_name, hostgroup):
+ """List all nodes of a hostgroup on stdout."""
+ prefix = '%s-%s' % (project_name, TOOLS_HOSTGROUPS[hostgroup])
+
+ api_url = 'https://wikitech.wikimedia.org/w/api.php' \
+ '?action=query&list=novainstances&niregion=eqiad&format=json' \
+ '&niproject=' + project_name
+
+ opener = urllib.request.build_opener()
+ opener.addheaders = [('User-Agent', __file__)]
+ data = json.loads(opener.open(api_url).read().decode('utf-8'))
+
+ for instance in data['query']['novainstances']:
+ name = instance['name']
+ if name.startswith(prefix):
+ print('%s.%s.eqiad.wmflabs' % (name, project_name))
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ subparsers = parser.add_subparsers(dest='action')
+
+ parser_map = subparsers.add_parser(
+ 'map',
+ help='Print list of hosts in a hostgroup'
+ )
+ parser_map.add_argument(
+ 'group',
+ help='Name of group whose instances should be printed'
+ )
+
+ parser_list = subparsers.add_parser(
+ 'list',
+ help='List all hostgroups',
+ )
+
+ args = parser.parse_args()
+ if args.action == 'list':
+ list_hostgroups()
+ elif args.action == 'map':
+ with open('/etc/wmflabs-project', 'r') as f:
+ project_name = f.read().rstrip('\n')
+
+ list_nodes(project_name, args.group)
diff --git a/modules/role/files/toollabs/clush/tools-clush-interpreter
b/modules/role/files/toollabs/clush/tools-clush-interpreter
deleted file mode 100644
index c2fb0a3..0000000
--- a/modules/role/files/toollabs/clush/tools-clush-interpreter
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/python3
-import yaml
-import argparse
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
- '--hostgroups',
- help='Path to YAML file with hostgroup information',
- required=True
-)
-subparsers = parser.add_subparsers(dest='action')
-
-parser_map = subparsers.add_parser(
- 'map',
- help='Print list of hosts in a hostgroup'
-)
-parser_map.add_argument(
- 'group',
- help='Name of group whose instances should be printed'
-)
-
-parser_list = subparsers.add_parser(
- 'list',
- help='List all hostgroups',
-)
-args = parser.parse_args()
-
-
-with open(args.hostgroups) as f:
- groups = yaml.safe_load(f)
- if args.action == 'map':
- print('\n'.join(groups[args.group]))
- if args.action == 'list':
- print('\n'.join(groups.keys()))
diff --git a/modules/role/manifests/toollabs/clush/master.pp
b/modules/role/manifests/toollabs/clush/master.pp
index eb869a2..0da9cd8 100644
--- a/modules/role/manifests/toollabs/clush/master.pp
+++ b/modules/role/manifests/toollabs/clush/master.pp
@@ -4,8 +4,7 @@
# command execution on classes of toollabs instances. Hits
# the wikitech API to do discovery of all instances in
# toollabs. They are then classified by prefix using a list,
-# maintained in modules/role/files/toollabs/clush/toollabs-clush-generator.
-# This is refreshed every hour.
+# maintained in modules/role/files/toollabs/clush/toollabs-clush-generator.py.
#
# You'll have to be a member of tools.admin to run this. All accesses
# are logged to /var/log/clush.log.
@@ -22,22 +21,17 @@
username => 'clushuser',
}
- require_package('python3-yaml')
-
file { '/usr/local/sbin/tools-clush-generator':
ensure => file,
- source =>
'puppet:///modules/role/toollabs/clush/tools-clush-generator',
+ source =>
'puppet:///modules/role/toollabs/clush/tools-clush-generator.py',
owner => 'root',
group => 'root',
mode => '0555',
}
+ # TODO: Remove after Puppet cycle.
file { '/usr/local/sbin/tools-clush-interpreter':
- ensure => file,
- source =>
'puppet:///modules/role/toollabs/clush/tools-clush-interpreter',
- owner => 'root',
- group => 'root',
- mode => '0555',
+ ensure => absent,
}
# override /usr/bin/clush with this! Just does additional logging
@@ -51,11 +45,14 @@
mode => '0555',
}
+ # TODO: Remove after Puppet cycle.
cron { 'update_tools_clush':
- ensure => present,
- command => '/usr/local/sbin/tools-clush-generator
/etc/clustershell/tools.yaml',
- hour => '*/1',
- user => 'root'
+ ensure => absent,
+ }
+
+ # TODO: Remove after Puppet cycle.
+ file { '/etc/clustershell/tools.yaml':
+ ensure => absent,
}
$groups_config = {
@@ -63,8 +60,8 @@
'default' => 'Tools',
},
'Tools' => {
- 'map' => '/usr/local/sbin/tools-clush-interpreter --hostgroups
/etc/clustershell/tools.yaml map $GROUP',
- 'list' => '/usr/local/sbin/tools-clush-interpreter --hostgroups
/etc/clustershell/tools.yaml list',
+ 'map' => '/usr/local/sbin/tools-clush-generator map $GROUP',
+ 'list' => '/usr/local/sbin/tools-clush-generator list',
}
}
--
To view, visit https://gerrit.wikimedia.org/r/328030
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5bb630836beb1c6a2aa2b346bef03687db9f4732
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Tim Landscheidt <t...@tim-landscheidt.de>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
