Ori.livneh has submitted this change and it was merged.
Change subject: add varnishstatsd
......................................................................
add varnishstatsd
varnishstatsd is a Python script that reports varnish backend response times
and varnish backend response codes to statsd. The key names it generates look
like this:
varnish.backends.{backend}.{status}
varnish.backends.{backend}.{method}
For example:
varnish.backends.ipv4_10_2_2_27.2xx:1|c
varnish.backends.ipv4_10_2_2_27.GET:48|ms
Change-Id: I5fb6808be0a72167a17440702f356e517ebae34e
---
A modules/varnish/files/varnishstatsd
A modules/varnish/manifests/logging/statsd.pp
A modules/varnish/templates/initscripts/varnishstatsd.systemd.erb
3 files changed, 216 insertions(+), 0 deletions(-)
Approvals:
Ori.livneh: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/varnish/files/varnishstatsd
b/modules/varnish/files/varnishstatsd
new file mode 100755
index 0000000..ca7e0ac
--- /dev/null
+++ b/modules/varnish/files/varnishstatsd
@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ varnishstatsd
+ ~~~~~~~~~~~~~
+ Report backend response times and request counts aggregated by status.
+
+ Copyright 2015 Ori Livneh <[email protected]>
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+"""
+import argparse
+import collections
+import io
+import socket
+import urlparse
+from varnishlog import varnishlog
+
+
+METRIC_FORMAT = (
+ '%(key_prefix)s%(backend)s.%(method)s:%(ttfb)d|ms\n'
+ '%(key_prefix)s%(backend)s.%(status)s:1|c\n'
+)
+
+UDP_MTU_BYTES = 1472
+
+vsl_args = [
+ ('i', 'Backend'),
+ ('i', 'BackendXID'),
+ ('i', 'ReqEnd'),
+ ('i', 'RxStatus'),
+ ('i', 'TxRequest'),
+]
+
+sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+
+
+def parse_statsd_server_string(server_string):
+ parsed = urlparse.urlparse('//' + server_string)
+ return parsed.hostname, parsed.port or 8125
+
+
+def parse_key_prefix(key_prefix):
+ return key_prefix.strip('.') + '.'
+
+
+ap = argparse.ArgumentParser(
+ description='Varnish backend response time metric logger',
+ epilog='If no statsd server is specified, prints stats to stdout instead.'
+)
+ap.add_argument('--varnish-name', help='varnish name')
+ap.add_argument('--statsd-server', help='statsd server',
+ type=parse_statsd_server_string, default=None)
+ap.add_argument('--key-prefix', help='metric key prefix',
+ default='varnish.backends',
+ type=parse_key_prefix)
+args = ap.parse_args()
+
+if args.varnish_name:
+ vsl_args.append(('n', args.varnish_name))
+
+# Maximum number of log records to keep in memory.
+MAX_BACKLOG = 100
+
+# Map of transaction ID: request method ('GET', 'POST', etc.)
+tids = collections.OrderedDict()
+
+# Map of backend XID: transaction ID
+xids = collections.OrderedDict()
+
+# Map of transaction ID: backend name
+backends = collections.OrderedDict()
+
+# Map of transaction ID: response status code (200, 404, etc.)
+statuses = collections.OrderedDict()
+
+buf = io.BytesIO()
+
+
+def vsl_callback(transaction_id, tag, record, remote_party):
+ global buf
+
+ if tag == 'Backend':
+ # Associate the transaction ID with the backend name.
+ tid, _, backend = record.split()
+ backends[int(tid)] = backend
+ if len(backends) > MAX_BACKLOG:
+ backends.popitem(last=False)
+
+ elif tag == 'BackendXID':
+ # Associate the backend XID with its transaction ID.
+ xids[record] = transaction_id
+ if len(xids) > MAX_BACKLOG:
+ xids.popitem(last=False)
+
+ elif tag == 'TxRequest' and remote_party == 'backend':
+ # Associate the transaction ID with the request method.
+ tids[transaction_id] = record
+ if len(tids) > MAX_BACKLOG:
+ tids.popitem(last=False)
+
+ elif tag == 'RxStatus' and remote_party == 'backend':
+ # Associate the transaction ID with the response status code.
+ statuses[transaction_id] = record[0] + 'xx'
+ if len(statuses) > MAX_BACKLOG:
+ statuses.popitem(last=False)
+
+ elif tag == 'ReqEnd':
+ # Collate data and emit metric.
+ parts = record.split()
+ xid = parts[0]
+
+ try:
+ tid = xids.pop(xid)
+ fields = {
+ 'key_prefix': args.key_prefix,
+ 'method': tids.pop(tid),
+ 'backend': backends.pop(tid),
+ 'status': statuses.pop(tid),
+ 'ttfb': round(1000 * float(parts[4])),
+ }
+ except KeyError:
+ return 0
+
+ metric_string = (METRIC_FORMAT % fields).encode('utf-8')
+ if buf.tell() + len(metric_string) >= UDP_MTU_BYTES:
+ buf.seek(io.SEEK_SET)
+ if args.statsd_server:
+ sock.sendto(buf.read(), args.statsd_server)
+ else:
+ print(buf.read().decode('utf-8', errors='replace').rstrip())
+ buf = io.BytesIO()
+ buf.write(metric_string)
+
+ return 0
+
+varnishlog(vsl_args, vsl_callback)
diff --git a/modules/varnish/manifests/logging/statsd.pp
b/modules/varnish/manifests/logging/statsd.pp
new file mode 100644
index 0000000..19db9f8
--- /dev/null
+++ b/modules/varnish/manifests/logging/statsd.pp
@@ -0,0 +1,59 @@
+# == Define: varnish::logging::statsd
+#
+# Report backend response time averages and response status
+# counts to StatsD.
+#
+# === Parameters
+#
+# [*instance_name*]
+# Varnish instance name. If not specified, monitor
+# the default instance.
+#
+# [*statsd_server*]
+# StatsD server address, in "host:port" format.
+# Defaults to localhost:8125.
+#
+# [*metric_prefix*]
+# A prefix to prepend to all metric names.
+# Defaults to "varnish.backends".
+#
+# === Examples
+#
+# varnish::logging::statsd { 'frontend':
+# instance_name => 'frontend
+# statsd_server => 'statsd.eqiad.wmnet:8125
+# metric_prefix => 'varnish.backends',
+# }
+#
+define varnish::logging::statsd(
+ $instance_name = '',
+ $statsd_server = 'statsd',
+ $metric_prefix = 'varnish.backends',
+) {
+ if $instance_name {
+ $service_unit_name = "varnishstatsd-${instance_name}"
+ } else {
+ $service_unit_name = "varnishstatsd-default"
+ }
+
+ if ! defined(File['/usr/local/bin/varnishstatsd']) {
+ file { '/usr/local/bin/varnishstatsd':
+ source => 'puppet:///modules/varnish/varnishstatsd',
+ owner => 'root',
+ group => 'root',
+ mode => '0555',
+ require =>
File['/usr/local/lib/python2.7/dist-packages/varnishlog.py'],
+ }
+ }
+
+ base::service_unit { $service_unit_name:
+ ensure => present,
+ systemd => true,
+ strict => false,
+ template_name => 'varnishstatsd',
+ require => File['/usr/local/bin/varnishstatsd'],
+ service_params => {
+ enable => true,
+ },
+ }
+}
diff --git a/modules/varnish/templates/initscripts/varnishstatsd.systemd.erb
b/modules/varnish/templates/initscripts/varnishstatsd.systemd.erb
new file mode 100644
index 0000000..dc21f1c
--- /dev/null
+++ b/modules/varnish/templates/initscripts/varnishstatsd.systemd.erb
@@ -0,0 +1,9 @@
+[Unit]
+Description=Varnish StatsD reporter
+After=varnish<%= /\w/.match(@instance_name) ? "-#{@instance_name}" : ''
-%>.service
+
+[Service]
+ExecStart=/usr/local/bin/varnishstatsd <%= /\w/.match(@instance_name) ?
"--varnish-instance_name=#{@instance_name}" : "" %> --statsd-server=<%=
@statsd_server %>
+
+[Install]
+WantedBy=multi-user.target
--
To view, visit https://gerrit.wikimedia.org/r/214147
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5fb6808be0a72167a17440702f356e517ebae34e
Gerrit-PatchSet: 8
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ori.livneh <[email protected]>
Gerrit-Reviewer: Alexandros Kosiaris <[email protected]>
Gerrit-Reviewer: BBlack <[email protected]>
Gerrit-Reviewer: Faidon Liambotis <[email protected]>
Gerrit-Reviewer: Filippo Giunchedi <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Ori.livneh <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
