Filippo Giunchedi has uploaded a new change for review. https://gerrit.wikimedia.org/r/144932
Change subject: swift: add swift-dispersion-report and stats ...................................................................... swift: add swift-dispersion-report and stats dispersion is a swift facility to assess the cluster health by writing containers and objects across the cluster once and subsequently verify where they are. http://docs.openstack.org/developer/swift/admin_guide.html add a swift dispersion account plus the reporting wrapper around swift-dispersion-report to push statistics to graphite (run by cron) dispersion needs to be primed (once, per cluster) via swift-dispersion-populate, this will be run upon cluster initialization manually. Change-Id: Icc323aa7103c8dca7c1ef5a407844523db8f5a09 --- A files/swift/swift-dispersion-stats M manifests/role/swift.pp M manifests/swift.pp A templates/swift/dispersion.conf.erb M templates/swift/proxy-server.conf.erb 5 files changed, 129 insertions(+), 4 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/32/144932/1 diff --git a/files/swift/swift-dispersion-stats b/files/swift/swift-dispersion-stats new file mode 100755 index 0000000..c4d5eba --- /dev/null +++ b/files/swift/swift-dispersion-stats @@ -0,0 +1,73 @@ +#!/usr/bin/python + +# report swift dispersion statistics, by default on stdout and optionally to +# a statsd server via UDP + +import argparse +import json +import os +import subprocess +import sys + +try: + import statsd + statsd_found = True +except ImportError: + statsd_found = False + + +def main(): + parser = argparse.ArgumentParser(description="Print swift dispersion statistics") + parser.add_argument('--prefix', dest='prefix', default='', + help='Prefix to use with the metrics') + parser.add_argument('--statsd-host', dest='statsd_host', default='', metavar="HOST", + help='Send metrics to this statsd host as well') + parser.add_argument('--statsd-port', dest='statsd_port', default='8125', + metavar="PORT", type=int, help='Send metrics to this statsd port') + args = parser.parse_args() + + dispersion_stats = {} + + try: + output = subprocess.check_output(['swift-dispersion-report', '-j']) + except subprocess.CalledProcessError, e: + print >>sys.stderr, 'swift-dispersion-report failed %s: %r' % ( + e.returncode, e.output) + return e.returncode + + try: + json_stats = json.loads(output) + except ValueError, e: + print >>sys.stderr, 'failed to load json from %r' % output + return 1 + + # {"object": { + # "retries": 0, "missing_0": 1304, "copies_expected": 2608, + # "pct_found": 100.0, "overlapping": 6, "copies_found": 2608 + # }, + # "container": { + # "retries": 0, "copies_expected": 2606, "pct_found": 100.0, + # "overlapping": 8, "copies_found": 2606 + # }} + for ring, stat in json_stats.iteritems(): + for name, value in stat.iteritems(): + key = '.'.join([args.prefix, ring, name]) + dispersion_stats[key] = value + + for key, value in dispersion_stats.iteritems(): + print "%s: %s" % (key, value) + + if args.statsd_host: + if not statsd_found: + print >>sys.stderr, "statsd module not found, unable to send" + return 1 + client = statsd.StatsClient(args.statsd_host, args.statsd_port) + for key, value in dispersion_stats.iteritems(): + try: + client.gauge(key, float(value)) + except ValueError: + print >>sys.stderr, "failed to send %r %r" % (key, value) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/manifests/role/swift.pp b/manifests/role/swift.pp index 61529a7..f31fb21 100644 --- a/manifests/role/swift.pp +++ b/manifests/role/swift.pp @@ -54,6 +54,22 @@ hour => "*", minute => "*", } + # swift-dispersion reporting + file { "/usr/local/bin/swift-dispersion-stats": + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + source => "puppet:///files/swift/swift-dispersion-stats", + require => [ Package['swift'], Package['python-statsd'] ], + } + cron { "swift-dispersion-stats": + ensure => present, + command => "/usr/local/bin/swift-dispersion-stats --prefix swift.eqiad-prod.dispersion --statsd-host statsd.eqiad.wmnet 1>/dev/null", + user => root, + hour => "*", + minute => "*/15", + } } class proxy inherits role::swift::eqiad-prod { class { "::swift::proxy": @@ -70,7 +86,8 @@ rewrite_password => $passwords::swift::eqiad_prod::rewrite_password, rewrite_thumb_server => "rendering.svc.eqiad.wmnet", shard_container_list => "wikipedia-commons-local-thumb,wikipedia-de-local-thumb,wikipedia-en-local-thumb,wikipedia-fi-local-thumb,wikipedia-fr-local-thumb,wikipedia-he-local-thumb,wikipedia-hu-local-thumb,wikipedia-id-local-thumb,wikipedia-it-local-thumb,wikipedia-ja-local-thumb,wikipedia-ro-local-thumb,wikipedia-ru-local-thumb,wikipedia-th-local-thumb,wikipedia-tr-local-thumb,wikipedia-uk-local-thumb,wikipedia-zh-local-thumb,wikipedia-commons-local-public,wikipedia-de-local-public,wikipedia-en-local-public,wikipedia-fi-local-public,wikipedia-fr-local-public,wikipedia-he-local-public,wikipedia-hu-local-public,wikipedia-id-local-public,wikipedia-it-local-public,wikipedia-ja-local-public,wikipedia-ro-local-public,wikipedia-ru-local-public,wikipedia-th-local-public,wikipedia-tr-local-public,wikipedia-uk-local-public,wikipedia-zh-local-public,wikipedia-commons-local-temp,wikipedia-de-local-temp,wikipedia-en-local-temp,wikipedia-fi-local-temp,wikipedia-fr-local-temp,wikipedia-he-local-temp,wikipedia-hu-local-temp,wikipedia-id-local-temp,wikipedia-it-local-temp,wikipedia-ja-local-temp,wikipedia-ro-local-temp,wikipedia-ru-local-temp,wikipedia-th-local-temp,wikipedia-tr-local-temp,wikipedia-uk-local-temp,wikipedia-zh-local-temp,wikipedia-commons-local-transcoded,wikipedia-de-local-transcoded,wikipedia-en-local-transcoded,wikipedia-fi-local-transcoded,wikipedia-fr-local-transcoded,wikipedia-he-local-transcoded,wikipedia-hu-local-transcoded,wikipedia-id-local-transcoded,wikipedia-it-local-transcoded,wikipedia-ja-local-transcoded,wikipedia-ro-local-transcoded,wikipedia-ru-local-transcoded,wikipedia-th-local-transcoded,wikipedia-tr-local-transcoded,wikipedia-uk-local-transcoded,wikipedia-zh-local-transcoded,global-data-math-render", - backend_url_format => "sitelang" + backend_url_format => "sitelang", + dispersion_password => $passwords::swift::eqiad_prod::dispersion_password, } class { '::swift::proxy::monitoring': host => 'ms-fe.eqiad.wmnet', @@ -130,6 +147,22 @@ hour => "*", minute => "*", } + # swift-dispersion reporting + file { "/usr/local/bin/swift-dispersion-stats": + ensure => present, + owner => 'root', + group => 'root', + mode => '0555', + source => "puppet:///files/swift/swift-dispersion-stats", + require => [ Package['swift'], Package['python-statsd'] ], + } + cron { "swift-dispersion-stats": + ensure => present, + command => "/usr/local/bin/swift-dispersion-stats --prefix swift.esams-prod.dispersion --statsd-host statsd.eqiad.wmnet 1>/dev/null", + user => root, + hour => "*", + minute => "*/15", + } } class proxy inherits role::swift::esams-prod { class { "::swift::proxy": @@ -146,7 +179,8 @@ rewrite_password => $passwords::swift::esams_prod::rewrite_password, rewrite_thumb_server => "upload.wikimedia.org", shard_container_list => "", - backend_url_format => "asis" + backend_url_format => "asis", + dispersion_password => $passwords::swift::esams_prod::dispersion_password, } class { '::swift::proxy::monitoring': host => 'ms-fe.esams.wmnet', @@ -214,7 +248,8 @@ rewrite_password => $passwords::swift::eqiad_prod::rewrite_password, rewrite_thumb_server => "rendering.svc.eqiad.wmnet", shard_container_list => "wikipedia-commons-local-thumb,wikipedia-de-local-thumb,wikipedia-en-local-thumb,wikipedia-fi-local-thumb,wikipedia-fr-local-thumb,wikipedia-he-local-thumb,wikipedia-hu-local-thumb,wikipedia-id-local-thumb,wikipedia-it-local-thumb,wikipedia-ja-local-thumb,wikipedia-ro-local-thumb,wikipedia-ru-local-thumb,wikipedia-th-local-thumb,wikipedia-tr-local-thumb,wikipedia-uk-local-thumb,wikipedia-zh-local-thumb,wikipedia-commons-local-public,wikipedia-de-local-public,wikipedia-en-local-public,wikipedia-fi-local-public,wikipedia-fr-local-public,wikipedia-he-local-public,wikipedia-hu-local-public,wikipedia-id-local-public,wikipedia-it-local-public,wikipedia-ja-local-public,wikipedia-ro-local-public,wikipedia-ru-local-public,wikipedia-th-local-public,wikipedia-tr-local-public,wikipedia-uk-local-public,wikipedia-zh-local-public,wikipedia-commons-local-temp,wikipedia-de-local-temp,wikipedia-en-local-temp,wikipedia-fi-local-temp,wikipedia-fr-local-temp,wikipedia-he-local-temp,wikipedia-hu-local-temp,wikipedia-id-local-temp,wikipedia-it-local-temp,wikipedia-ja-local-temp,wikipedia-ro-local-temp,wikipedia-ru-local-temp,wikipedia-th-local-temp,wikipedia-tr-local-temp,wikipedia-uk-local-temp,wikipedia-zh-local-temp,wikipedia-commons-local-transcoded,wikipedia-de-local-transcoded,wikipedia-en-local-transcoded,wikipedia-fi-local-transcoded,wikipedia-fr-local-transcoded,wikipedia-he-local-transcoded,wikipedia-hu-local-transcoded,wikipedia-id-local-transcoded,wikipedia-it-local-transcoded,wikipedia-ja-local-transcoded,wikipedia-ro-local-transcoded,wikipedia-ru-local-transcoded,wikipedia-th-local-transcoded,wikipedia-tr-local-transcoded,wikipedia-uk-local-transcoded,wikipedia-zh-local-transcoded,global-data-math-render", - backend_url_format => "sitelang" + backend_url_format => "sitelang", + dispersion_password => $passwords::swift::eqiad_prod::dispersion_password, } class { '::swift::proxy::monitoring': host => $swift_proxy_hostname, diff --git a/manifests/swift.pp b/manifests/swift.pp index b2d0147..bd140c8 100644 --- a/manifests/swift.pp +++ b/manifests/swift.pp @@ -123,7 +123,8 @@ $rewrite_password, $rewrite_thumb_server, $shard_container_list, - $backend_url_format + $backend_url_format, + $dispersion_password, ) { Class['swift::base'] -> Class['swift::proxy'] @@ -139,6 +140,14 @@ require => Package['swift-proxy'], } + file { '/etc/swift/dispersion.conf': + owner => 'swift', + group => 'swift', + mode => '0440', + content => template('swift/dispersion.conf.erb'), + require => Package['swift'], + } + package {[ 'swift-proxy', 'python-swauth' diff --git a/templates/swift/dispersion.conf.erb b/templates/swift/dispersion.conf.erb new file mode 100644 index 0000000..876cb16 --- /dev/null +++ b/templates/swift/dispersion.conf.erb @@ -0,0 +1,7 @@ +# This file is managed by Puppet! + +[dispersion] +auth_url = http://<%= @proxy_address %>/auth/v1.0 +auth_user = swift:dispersion +auth_key = <%= @dispersion_password %> +swift_dir = /etc/swift diff --git a/templates/swift/proxy-server.conf.erb b/templates/swift/proxy-server.conf.erb index f6e513c..17f8853 100644 --- a/templates/swift/proxy-server.conf.erb +++ b/templates/swift/proxy-server.conf.erb @@ -46,6 +46,7 @@ token_life = 604800 user_admin_admin = <%= @super_admin_key %> .admin .reseller_admin <%= @proxy_address %>/v1/AUTH_admin user_mw_media = <%= @rewrite_password %> .admin <%= @proxy_address %>/v1/AUTH_mw +user_swift_dispersion = <%= @dispersion_password %> .admin <%= @proxy_address %>/v1/AUTH_dispersion <% end -%> [filter:healthcheck] -- To view, visit https://gerrit.wikimedia.org/r/144932 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Icc323aa7103c8dca7c1ef5a407844523db8f5a09 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Filippo Giunchedi <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
