Ema has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/398819 )
Change subject: mtail: add varnishreqstats.mtail
......................................................................
mtail: add varnishreqstats.mtail
Add an mtail script introducing a new metric called `varnish_requests`.
The metric counts HTTP request methods and response status codes.
Bug: T177199
Change-Id: I85e362d434acaf941e46c485ef1aed70a74165dc
---
M modules/mtail/files/programs/varnishmedia.mtail
A modules/mtail/files/programs/varnishreqstats.mtail
M modules/mtail/files/test/logs/varnish.test
M modules/mtail/files/test/varnish_test.py
M modules/varnish/files/varnishmtail
5 files changed, 31 insertions(+), 13 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/19/398819/1
diff --git a/modules/mtail/files/programs/varnishmedia.mtail
b/modules/mtail/files/programs/varnishmedia.mtail
index 885af9e..30325a0 100644
--- a/modules/mtail/files/programs/varnishmedia.mtail
+++ b/modules/mtail/files/programs/varnishmedia.mtail
@@ -1,5 +1,5 @@
counter varnish_thumbnails by status
-/^url .*\/thumb\/.*\thttp_status (?P<http_status>.*)\tcache_control/ {
+/^url .*\/thumb\/.*\thttp_status (?P<http_status>[1-5][0-9][0-9])\t/ {
varnish_thumbnails[$http_status]++
}
diff --git a/modules/mtail/files/programs/varnishreqstats.mtail
b/modules/mtail/files/programs/varnishreqstats.mtail
new file mode 100644
index 0000000..12c4d0e
--- /dev/null
+++ b/modules/mtail/files/programs/varnishreqstats.mtail
@@ -0,0 +1,5 @@
+counter varnish_requests by status, method
+
+/\thttp_status (?P<status>[1-5][0-9][0-9])\thttp_method (?P<method>[A-Z]+)\t/ {
+ varnish_requests[$status][$method]++
+}
diff --git a/modules/mtail/files/test/logs/varnish.test
b/modules/mtail/files/test/logs/varnish.test
index 5cd96e5..a8f0da0 100644
--- a/modules/mtail/files/test/logs/varnish.test
+++ b/modules/mtail/files/test/logs/varnish.test
@@ -1,9 +1,9 @@
-url / cache_status int-front http_status 301 cache_control - inm - h2 0
tls_version session_reused 0 key_exchange auth cipher
full_cipher
-url /w/index.php cache_status hit-front http_status 304 cache_control
private, s-maxage=0, max-age=0, must-revalidate inm - h2 1 tls_version
TLSv1.2 session_reused 1 key_exchange X25519 auth ECDSA
cipher CHACHA20-POLY1305 full_cipher ECDHE-ECDSA-CHACHA20-POLY1305
-url
/api/rest_v1/metrics/pageviews/per-article/fr.wikipedia.org/all-access/user/S%C3%A9lection_sexuelle/daily/20171126/20171201
cache_status hit-front http_status 200 cache_control s-maxage=86400,
max-age=86400 inm -
-url /w/load.php cache_status hit-front http_status 200 cache_control
public, max-age=2592000, s-maxage=2592000 inm -
-url /w/load.php cache_status hit-front http_status 200 cache_control
public, max-age=2592000, s-maxage=2592000 inm -
-url /w/load.php cache_status hit-front http_status 304 cache_control
public, max-age=300, s-maxage=300 inm W/\"1adp3u3\"
-url /wikipedia/commons/5/51/Tang_Shaoyi.jpg cache_status int-front
http_status 301 cache_control - inm -
-url
/wikipedia/commons/thumb/5/51/Flag_of_North_Korea.svg/250px-Flag_of_North_Korea.svg.png
cache_status hit-front http_status 200 cache_control - inm -
-url /wikipedia/en/thumb/f/fd/Portal-puzzle.svg/16px-Portal-puzzle.svg.png
cache_status hit-front http_status 200 cache_control - inm -
+url / cache_status int-front http_status 301 http_method GET cache_control -
inm - h2 0 tls_version session_reused 0 key_exchange auth
cipher full_cipher
+url /w/index.php cache_status hit-front http_status 304 http_method GET
cache_control private, s-maxage=0, max-age=0, must-revalidate inm - h2 1
tls_version TLSv1.2 session_reused 1 key_exchange X25519 auth
ECDSA cipher CHACHA20-POLY1305 full_cipher
ECDHE-ECDSA-CHACHA20-POLY1305
+url
/api/rest_v1/metrics/pageviews/per-article/fr.wikipedia.org/all-access/user/S%C3%A9lection_sexuelle/daily/20171126/20171201
cache_status hit-front http_status 200 http_method HEAD
cache_control s-maxage=86400, max-age=86400 inm -
+url /w/load.php cache_status hit-front http_status 200 http_method GET
cache_control public, max-age=2592000, s-maxage=2592000 inm -
+url /w/load.php cache_status hit-front http_status 200 http_method
HEAD cache_control public, max-age=2592000, s-maxage=2592000 inm -
+url /w/load.php cache_status hit-front http_status 304 http_method GET
cache_control public, max-age=300, s-maxage=300 inm W/\"1adp3u3\"
+url /wikipedia/commons/5/51/Tang_Shaoyi.jpg cache_status int-front
http_status 301 http_method GET cache_control - inm -
+url
/wikipedia/commons/thumb/5/51/Flag_of_North_Korea.svg/250px-Flag_of_North_Korea.svg.png
cache_status hit-front http_status 200 http_method GET cache_control - inm
-
+url /wikipedia/en/thumb/f/fd/Portal-puzzle.svg/16px-Portal-puzzle.svg.png
cache_status hit-front http_status 200 http_method GET cache_control - inm -
diff --git a/modules/mtail/files/test/varnish_test.py
b/modules/mtail/files/test/varnish_test.py
index 40c6fc4..c22e198 100644
--- a/modules/mtail/files/test/varnish_test.py
+++ b/modules/mtail/files/test/varnish_test.py
@@ -66,3 +66,16 @@
self.assertIn(value, labels)
self.assertEquals(1, count)
+
+
+class VarnishReqStatsTest(unittest.TestCase):
+ def setUp(self):
+ self.store = mtail_store.MtailMetricStore(
+ os.path.join(test_dir, '../programs/varnishreqstats.mtail'),
+ os.path.join(test_dir, 'logs/varnish.test'))
+
+ def testRespStatus(self):
+ s = self.store.get_samples('varnish_requests')
+ self.assertIn(('status=200,method=GET', 3), s)
+ self.assertIn(('status=301,method=GET', 2), s)
+ self.assertIn(('status=200,method=HEAD', 2), s)
diff --git a/modules/varnish/files/varnishmtail
b/modules/varnish/files/varnishmtail
index ba2de95..5ffe9cd 100644
--- a/modules/varnish/files/varnishmtail
+++ b/modules/varnish/files/varnishmtail
@@ -7,6 +7,7 @@
fmt_url='url %U'
fmt_cache_status='cache_status %{X-Cache-Status}o'
fmt_http_status='http_status %s'
+fmt_http_method='http_method %m'
fmt_cache_control='cache_control %{Cache-Control}o'
fmt_inm='inm %{If-None-Match}i'
fmt_h2='h2 %{VCL_Log:CP-HTTP2}x'
@@ -17,7 +18,6 @@
fmt_cipher='cipher %{VCL_Log:CP-Cipher}x'
fmt_full_cipher='full_cipher %{VCL_Log:CP-Full-Cipher}x'
-FMT="${fmt_url}\t${fmt_cache_status}\t${fmt_http_status}\t${fmt_cache_control}\t${fmt_inm}\t${fmt_h2}\t${fmt_tls_version}\t${fmt_session_reused}\t${fmt_key_exchange}\t${fmt_auth}\t${fmt_cipher}\t${fmt_full_cipher}\t"
+FMT="${fmt_url}\t${fmt_cache_status}\t${fmt_http_status}\t${fmt_http_method}\t${fmt_cache_control}\t${fmt_inm}\t${fmt_h2}\t${fmt_tls_version}\t${fmt_session_reused}\t${fmt_key_exchange}\t${fmt_auth}\t${fmt_cipher}\t${fmt_full_cipher}\t"
-/usr/bin/varnishncsa -n frontend -q 'ReqMethod ne "PURGE"' -F "${FMT}" |
- mtail -progs "${PROGS}" -logfds 0
+/usr/bin/varnishncsa -n frontend -F "${FMT}" | mtail -progs "${PROGS}" -logfds 0
--
To view, visit https://gerrit.wikimedia.org/r/398819
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I85e362d434acaf941e46c485ef1aed70a74165dc
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ema <e...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
