Filippo Giunchedi has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/350817 )
Change subject: [WIP] Send 5xx from kafkatee to logstash
......................................................................
[WIP] Send 5xx from kafkatee to logstash
Straw man patch to send 5xx read from webrequest kafka topics with kafkatee to
logstash.
The 'type' field is used to distinguish existing 'json_lines' filters and be
able to further filter webrequest in the future (e.g. to redact fields)
Bug: T149451
Change-Id: I451160b0089daf8242a8b49737052f61300b1e7e
---
M hieradata/role/common/logging/kafkatee/webrequest/ops.yaml
M modules/role/manifests/logging/kafkatee/webrequest/ops.pp
M modules/role/manifests/logstash/collector.pp
3 files changed, 22 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/17/350817/1
diff --git a/hieradata/role/common/logging/kafkatee/webrequest/ops.yaml
b/hieradata/role/common/logging/kafkatee/webrequest/ops.yaml
index 43d013a..e96ae06 100644
--- a/hieradata/role/common/logging/kafkatee/webrequest/ops.yaml
+++ b/hieradata/role/common/logging/kafkatee/webrequest/ops.yaml
@@ -1,3 +1,5 @@
debdeploy::grains:
debdeploy-kafkatee:
value: standard
+role::logging::kafkatee::webrequest::ops::logstash_host:
logstash1001.eqiad.wmnet
+role::logging::kafkatee::webrequest::ops::logstash_port: 11515
diff --git a/modules/role/manifests/logging/kafkatee/webrequest/ops.pp
b/modules/role/manifests/logging/kafkatee/webrequest/ops.pp
index 694e6f2..015a662 100644
--- a/modules/role/manifests/logging/kafkatee/webrequest/ops.pp
+++ b/modules/role/manifests/logging/kafkatee/webrequest/ops.pp
@@ -6,6 +6,8 @@
include role::logging::kafkatee::webrequest::base
$webrequest_log_directory =
$::role::logging::kafkatee::webrequest::base::webrequest_log_directory
+ $logstash_host =
hiera('role::logging::kafkatee::webrequest::ops::logstash_host')
+ $logstash_port =
hiera('role::logging::kafkatee::webrequest::ops::logstash_port')
kafkatee::output { 'sampled-1000':
destination => "${webrequest_log_directory}/sampled-1000.json",
@@ -19,4 +21,10 @@
destination => "/bin/grep --line-buffered '\"http_status\":\"5' >>
${webrequest_log_directory}/5xx.json",
type => 'pipe',
}
+
+ # Send 5xx to logstash, append "type: webrequest" for logstash to pick up
+ kafkatee::output { 'logstash-5xx':
+ destination => "/bin/grep --line-buffered '\"http_status\":\"5' | jq
--arg type webrequest '. + {type: \$type}' | socat -
TCP:$logstash_host:$logstash_port",
+ type => 'pipe',
+ }
}
diff --git a/modules/role/manifests/logstash/collector.pp
b/modules/role/manifests/logstash/collector.pp
index 7d4952c..391df8d 100644
--- a/modules/role/manifests/logstash/collector.pp
+++ b/modules/role/manifests/logstash/collector.pp
@@ -100,6 +100,18 @@
srange => '$DOMAIN_NETWORKS',
}
+ logstash::input::tcp { 'webrequest':
+ port => 11515,
+ codec => 'json_lines',
+ }
+
+ ferm::service { 'logstash_webrequest':
+ proto => 'tcp',
+ port => '11515',
+ notrack => true,
+ srange => '$DOMAIN_NETWORKS',
+ }
+
## Global pre-processing (15)
# move files into module?
--
To view, visit https://gerrit.wikimedia.org/r/350817
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I451160b0089daf8242a8b49737052f61300b1e7e
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Filippo Giunchedi <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
