Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/94169
Change subject: Setting up varnishkafka on mobile varnish caches.
......................................................................
Setting up varnishkafka on mobile varnish caches.
Change-Id: I23d545e48b9046dd6d54df2281b3d76c82758771
---
M .gitmodules
M manifests/role/cache.pp
A modules/varnishkafka
3 files changed, 33 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/69/94169/1
diff --git a/.gitmodules b/.gitmodules
index a5b3306..a78db6c 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -10,3 +10,6 @@
[submodule "modules/kafka"]
path = modules/kafka
url = https://gerrit.wikimedia.org/r/p/operations/puppet/kafka
+[submodule "modules/varnishkafka"]
+ path = modules/varnishkafka
+ url = https://gerrit.wikimedia.org/r/operations/puppet/varnishkafka
diff --git a/manifests/role/cache.pp b/manifests/role/cache.pp
index 7d5ff00..389dc9d 100644
--- a/manifests/role/cache.pp
+++ b/manifests/role/cache.pp
@@ -465,6 +465,33 @@
}
}
+ # == Class varnish::kafka
+ # Sets up a varnishkafka instance producing varnish
+ # logs to the analytics Kafka brokers in eqiad.
+ class varnish::kafka($topic)
+ {
+ if $::realm == 'production' {
+ require role::analytics::kafka::config
+ $kafka_brokers =
keys($role::analytics::kafka::config::brokers)
+
+ class { '::varnishkafka':
+ brokers => $kafka_brokers,
+ topic => $topic,
+ format_type => 'json',
+ # Note: statictag tricks varnishkafka into
allowing hardcoded string into a json field.
+ # Hardcoding the $fqdn into hostname rather
than using %l to account for
+ # possible slip ups where varnish only writes
the short hostname for %l.
+ format =>
"%{statictag0@hostname?${::fqdn}}x %{@sequence!num?0}n %{%FT%T@dt}t
%{Varnish:time_firstbyte@time_firstbyte!num?0.0}x %{@ip}h
%{Varnish:handling@cache_status}x %{@http_status}s %{@response_size!num?0}b
%{@http_method}m %{Host@uri_host}i %{@uri_path}U %{@uri_query}q
%{Content-Type@content_type}o %{Referer@referer}i
%{X-Forwarded-For@x_forwarded_for}i %{User-Agent@user_agent}i
%{Accept-Language@accept_language}i %{X-Analytics@x_analytics}i",
+ format_key_type => 'string',
+ # Kafka partion key is machine hostname
+ format_key => "${::fqdn}",
+ # 3 minutes, large to account for potential
cross DC latencies
+ topic_message_timeout_ms => 180000,
+ compression_codec => 'snappy',
+ }
+ }
+ }
+
class varnish::logging::eventlistener {
if $::realm == 'production' {
$event_listener = $::site ? {
@@ -1096,7 +1123,9 @@
cluster_options => $cluster_options,
}
- include role::cache::varnish::logging
+ class { 'role::cache::varnish::kafka':
+ topic => 'varnish-mobile'
+ }
}
class parsoid inherits role::cache::varnish::2layer {
diff --git a/modules/varnishkafka b/modules/varnishkafka
new file mode 160000
index 0000000..43e159a
--- /dev/null
+++ b/modules/varnishkafka
+Subproject commit 43e159afbca3ed4f46c666a2b2f7106869cbe78a
--
To view, visit https://gerrit.wikimedia.org/r/94169
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I23d545e48b9046dd6d54df2281b3d76c82758771
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
