Filippo Giunchedi has uploaded a new change for review.
https://gerrit.wikimedia.org/r/240358
Change subject: WIP: report swift containers aggregated stats
......................................................................
WIP: report swift containers aggregated stats
Change-Id: Icb7605dbb334ccf225a05ffe54370b33decd090c
---
A modules/swift/files/swift-container-stats
M modules/swift/manifests/stats/accounts.pp
A modules/swift/manifests/stats/stats_container.pp
3 files changed, 145 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/58/240358/1
diff --git a/modules/swift/files/swift-container-stats
b/modules/swift/files/swift-container-stats
new file mode 100755
index 0000000..fcbbd60
--- /dev/null
+++ b/modules/swift/files/swift-container-stats
@@ -0,0 +1,117 @@
+#!/usr/bin/python
+
+# report swift container statistics, by default on stdout and optionally to
+# a statsd server via UDP.
+
+import argparse
+import os
+import re
+import sys
+
+import swiftclient
+
+try:
+ import statsd
+ statsd_found = True
+except ImportError:
+ statsd_found = False
+
+
+# map a container set name to a dict of:
+# container_regexp: bucket_name
+# each container matching the given regex will have its stats reported under
bucket_name
+CONTAINER_SETS = {
+ # MediaWiki containers used for media storage
+ 'mw-media': {
+ re.compile('-thumb(\.[a-z0-9][a-z0-9])?$'): 'thumb',
+ re.compile('-public(\.[a-z0-9][a-z0-9])?$'): 'originals',
+ re.compile('-temp(\.[a-z0-9][a-z0-9])?$'): 'temp',
+ re.compile('-deleted(\.[a-z0-9][a-z0-9])?$'): 'deleted',
+ re.compile('-transcoded(\.[a-z0-9][a-z0-9])?$'): 'transcoded',
+ re.compile('-render(\.[a-z0-9][a-z0-9])?$'): 'render',
+ },
+}
+
+
+def container_bucket(name, bucket_map):
+ for container_re, bucket in bucket_map.iteritems():
+ if container_re.search(name):
+ return bucket
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Print swift container
statistics")
+ parser.add_argument('-A', '--auth', dest='auth',
+ default=os.environ.get('ST_AUTH', None),
+ help='URL for obtaining an auth token')
+ parser.add_argument('-U', '--user', dest='user',
+ default=os.environ.get('ST_USER', None),
+ help='User name for obtaining an auth token')
+ parser.add_argument('-K', '--key', dest='key',
+ default=os.environ.get('ST_KEY', None),
+ help='Key for obtaining an auth token')
+ parser.add_argument('--prefix', dest='prefix',
+ default='',
+ help='Prefix to use when reporting metrics')
+ parser.add_argument('--container-set', dest='container_set',
+ default='', choices=CONTAINER_SETS.keys(),
+ help='Report aggregated container statistics from this set')
+ parser.add_argument('--ignore-unknown', dest='ignore_unknown',
+ default=False, action='store_true',
+ help='Do not report unknown containers')
+ parser.add_argument('--statsd-host', dest='statsd_host',
+ default='', metavar="HOST",
+ help='Send metrics to this statsd host as well')
+ parser.add_argument('--statsd-port', dest='statsd_port',
+ default='8125', metavar="PORT", type=int,
+ help='Send metrics to this statsd port')
+ args = parser.parse_args()
+
+ if None in (args.auth, args.user, args.key):
+ parser.error("please provide auth, user and key")
+ return 1
+
+ if not args.container_set:
+ parser.error("please provide a container set")
+ return 1
+
+ container_stats = {}
+ output_stats = []
+ container_buckets = CONTAINER_SETS[args.container_set]
+
+ connection = swiftclient.Connection(args.auth, args.user, args.key)
+ headers, containers = connection.get_account(full_listing=True)
+ for container in containers:
+ bucket = container_bucket(container['name'], container_buckets)
+ if bucket is None:
+ if not args.ignore_unknown:
+ print >>sys.stderr, "Cannot find bucket for container %r" %
container['name']
+ continue
+
+ bucket_stats = container_stats.setdefault(
+ bucket, {'bytes': 0, 'objects': 0})
+ bucket_stats['bytes'] += container['bytes']
+ bucket_stats['objects'] += container['count']
+
+ for bucket, stats in container_stats.iteritems():
+ for stat in ('bytes', 'objects'):
+ prefix = '.'.join([args.prefix, bucket, stat])
+ output_stats.append((prefix, stats[stat]))
+
+ for name, value in output_stats:
+ print "%s: %s" % (name, value)
+
+ if args.statsd_host:
+ if not statsd_found:
+ print >>sys.stderr, "statsd module not found, unable to send"
+ return 1
+ client = statsd.StatsClient(args.statsd_host, args.statsd_port)
+ for name, value in output_stats:
+ try:
+ client.gauge(name, float(value))
+ except ValueError:
+ print >>sys.stderr, "failed to send %r %r" % (name, value)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/modules/swift/manifests/stats/accounts.pp
b/modules/swift/manifests/stats/accounts.pp
index 7cb8096..59de325 100644
--- a/modules/swift/manifests/stats/accounts.pp
+++ b/modules/swift/manifests/stats/accounts.pp
@@ -21,6 +21,16 @@
require => $required_packages,
}
+ # report container stats to graphite
+ file { '/usr/local/bin/swift-container-stats':
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ source => 'puppet:///modules/swift/swift-container-stats',
+ require => $required_packages,
+ }
+
$account_names = sort(keys($accounts))
swift::stats::stats_account { $account_names:
accounts => $accounts,
diff --git a/modules/swift/manifests/stats/stats_container.pp
b/modules/swift/manifests/stats/stats_container.pp
new file mode 100644
index 0000000..2b667d2
--- /dev/null
+++ b/modules/swift/manifests/stats/stats_container.pp
@@ -0,0 +1,18 @@
+define swift::stats::stats_container (
+ $account_name,
+ $statsd_prefix,
+ $statsd_host,
+) {
+ $account_file = "/etc/swift/account_${account_name}.env"
+ $container_statsd_prefix = "${statsd_prefix}.${account_name}"
+
+ cron { "swift-container-stats_${account_name}":
+ ensure => present,
+ command => ". ${account_file} && /usr/local/bin/swift-container-stats
--prefix ${container_statsd_prefix} --statsd-host ${statsd_host}
--ignore-unknown 1>/dev/null",
+ user => 'root',
+ hour => '*',
+ minute => '*',
+ require => [File[$account_file],
+ File['/usr/local/bin/swift-container-stats']],
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/240358
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icb7605dbb334ccf225a05ffe54370b33decd090c
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Filippo Giunchedi <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
