[MediaWiki-commits] [Gerrit] operations/puppet[production]: Smarter Varnish slow log
Ema has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/399176 )
Change subject: Smarter Varnish slow log
..
Smarter Varnish slow log
Using python-logstash allows us to send structured data
to ELK. Additionally, we can leverage varnishlog, which
has more options than varnishncsa.
Bug: T181315
Change-Id: I08851a84857783cfacc75768a3c0216633aa9242
---
M modules/profile/manifests/cache/base.pp
A modules/varnish/files/varnishslowlog.py
M modules/varnish/manifests/common.pp
M modules/varnish/manifests/instance.pp
M modules/varnish/manifests/logging.pp
D modules/varnish/templates/initscripts/varnish-slowreqs.systemd.erb
A modules/varnish/templates/initscripts/varnishslowlog.systemd.erb
7 files changed, 193 insertions(+), 25 deletions(-)
Approvals:
Ema: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/profile/manifests/cache/base.pp
b/modules/profile/manifests/cache/base.pp
index 6fc84a6..179a534 100644
--- a/modules/profile/manifests/cache/base.pp
+++ b/modules/profile/manifests/cache/base.pp
@@ -22,6 +22,7 @@
$be_runtime_params = hiera('profile::cache::base::be_runtime_params', []),
$logstash_host = hiera('logstash_host', undef),
$logstash_syslog_port = hiera('logstash_syslog_port', undef),
+$logstash_json_lines_port = hiera('logstash_json_lines_port', undef),
$log_slow_request_threshold =
hiera('profile::cache::base::log_slow_request_threshold', '60.0'),
$allow_iptables = hiera('profile::cache::base::allow_iptables', false),
) {
@@ -78,6 +79,8 @@
fe_runtime_params => $fe_runtime_params,
be_runtime_params => $be_runtime_params,
log_slow_request_threshold => $log_slow_request_threshold,
+logstash_host => $logstash_host,
+logstash_json_lines_port => $logstash_json_lines_port,
}
class { [
diff --git a/modules/varnish/files/varnishslowlog.py
b/modules/varnish/files/varnishslowlog.py
new file mode 100644
index 000..349e347
--- /dev/null
+++ b/modules/varnish/files/varnishslowlog.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ VarnishSlowLog
+ ~~
+ VarnishSlowLog is responsible for gathering slow requests from varnishlog
+ and sending them to logstash.
+
+ Copyright 2016-2017 Emanuele Rocca
+ Copyright 2017 Gilles Dubuc
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+"""
+
+import argparse
+import logging
+import logstash
+import os
+import sys
+import urlparse
+
+from subprocess import PIPE, Popen
+
+
+def parse_logstash_server_string(server_string):
+"""Convert logstash server string into (hostname, port) tuple."""
+parsed = urlparse.urlparse('//' + server_string)
+return parsed.hostname, parsed.port or 12202
+
+
+class VarnishSlowLog(object):
+description = 'Varnish slow log logstash logger'
+
+def __init__(self, argument_list):
+"""Parse CLI arguments.
+
+argument_list is a list such as ['--foo', 'FOO', '--bar', 'BAR']"""
+ap = argparse.ArgumentParser(
+description=self.description,
+epilog='If no logstash server is specified, '
+ 'prints log entries to stdout instead.')
+
+ap.add_argument('--logstash-server', help='logstash server',
+type=parse_logstash_server_string, default=None)
+
+ap.add_argument('--slow-threshold', help='varnish fetch duration
threshold',
+type=float, default=10.0)
+
+ap.add_argument('--transaction-timeout', help='varnish transaction
timeout',
+type=int, default=600)
+
+ap.add_argument('--varnishd-instance-name', help='varnishd instance
name',
+default=None)
+
+ap.add_argument('--varnishlog-path', help='varnishlog full path',
+default='/usr/bin/varnishlog')
+
+self.args = ap.parse_args(argument_list)
+
+self.cmd = [
+self.args.varnishlog_path,
+# VSL query matching anything but purges
+'-q', 'ReqMethod ne "PURGE" and Timestamp:Fetch[3] > %f' %
self.args.slow_threshold,
+# Set maximum Varnish transaction duration to track
+'-T', '%d' % self.args.transaction_timeout
+]
+
+self.layer = 'backend'
+
+if
[MediaWiki-commits] [Gerrit] operations/puppet[production]: Smarter Varnish slow log
Gilles has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/399176 )
Change subject: Smarter Varnish slow log
..
Smarter Varnish slow log
Using python-logstash allows us to send structured data
to ELK. Additionally, we can leverage varnishlog, which
has more options than varnishncsa.
Bug: T181315
Change-Id: I08851a84857783cfacc75768a3c0216633aa9242
---
M hieradata/common.yaml
M modules/profile/manifests/cache/base.pp
A modules/varnish/files/varnishslowlog.py
M modules/varnish/manifests/common.pp
M modules/varnish/manifests/instance.pp
D modules/varnish/templates/initscripts/varnish-slowreqs.systemd.erb
A modules/varnish/templates/initscripts/varnishslowlog.systemd.erb
7 files changed, 189 insertions(+), 19 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/76/399176/1
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 4ff228a..7349b31 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -520,6 +520,7 @@
logstash_host: "logstash.svc.eqiad.wmnet"
logstash_syslog_port: 10514
logstash_gelf_port: 12201
+logstash_json_port: 11514
udp2log_aggregator: "udplog:8420"
tcpircbot_host: 'icinga.wikimedia.org'
diff --git a/modules/profile/manifests/cache/base.pp
b/modules/profile/manifests/cache/base.pp
index 6fc84a6..a8508ff 100644
--- a/modules/profile/manifests/cache/base.pp
+++ b/modules/profile/manifests/cache/base.pp
@@ -22,6 +22,7 @@
$be_runtime_params = hiera('profile::cache::base::be_runtime_params', []),
$logstash_host = hiera('logstash_host', undef),
$logstash_syslog_port = hiera('logstash_syslog_port', undef),
+$logstash_json_port = hiera('logstash_json_port', undef),
$log_slow_request_threshold =
hiera('profile::cache::base::log_slow_request_threshold', '60.0'),
$allow_iptables = hiera('profile::cache::base::allow_iptables', false),
) {
@@ -78,6 +79,8 @@
fe_runtime_params => $fe_runtime_params,
be_runtime_params => $be_runtime_params,
log_slow_request_threshold => $log_slow_request_threshold,
+logstash_host => $logstash_host,
+logstash_json_port => $logstash_json_port,
}
class { [
diff --git a/modules/varnish/files/varnishslowlog.py
b/modules/varnish/files/varnishslowlog.py
new file mode 100644
index 000..588fc06
--- /dev/null
+++ b/modules/varnish/files/varnishslowlog.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ VarnishSlowLog
+ ~~
+ VarnishSlowLog is responsible for gathering slow requests from varnishlog
+ and sending them to logstash.
+
+ Copyright 2016-2017 Emanuele Rocca
+ Copyright 2017 Gilles Dubuc
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+"""
+
+import argparse
+import logging
+import logstash
+import os
+import sys
+import urlparse
+
+from subprocess import PIPE, Popen
+
+
+def parse_logstash_server_string(server_string):
+"""Convert logstash server string into (hostname, port) tuple."""
+parsed = urlparse.urlparse('//' + server_string)
+return parsed.hostname, parsed.port or 12202
+
+
+class VarnishSlowLog(object):
+description = 'Varnish slow log logstash logger'
+
+def __init__(self, argument_list):
+"""Parse CLI arguments.
+
+argument_list is a list such as ['--foo', 'FOO', '--bar', 'BAR']"""
+ap = argparse.ArgumentParser(
+description=self.description,
+epilog='If no logstash server is specified, '
+ 'prints log entries to stdout instead.')
+
+ap.add_argument('--logstash-server', help='logstash server',
+type=parse_logstash_server_string, default=None)
+
+ap.add_argument('--slow-threshold', help='varnish fetch duration
threshold',
+type=float, default=10.0)
+
+ap.add_argument('--transaction-timeout', help='varnish transaction
timeout',
+type=int, default=600)
+
+ap.add_argument('--varnishd-instance-name', help='varnishd instance
name',
+default=None)
+
+ap.add_argument('--varnishlog-path', help='varnishlog full path',
+default='/usr/bin/varnishlog')
+
+self.args = ap.parse_args(argument_list)
+
+self.cmd = [self.args.varnishlog_path,
+
