BryanDavis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/227175
Change subject: labs: new role::logstash::stashbot class
......................................................................
labs: new role::logstash::stashbot class
Add a new role and configuration for use in the logstash labs project to
create an instance for logging irc messages and SAL !log data.
Change-Id: Ia04219138f3dab667d7c2a667994681aeeabc563
---
D files/logstash/filter-irc-banglog.conf
A files/logstash/filter-stashbot-sal.conf
A files/logstash/filter-stashbot.conf
A files/logstash/stashbot-sal-template.json
A files/logstash/stashbot-template.json
A hieradata/labs/logstash/host/stashbot01.yaml
M manifests/role/logstash.pp
7 files changed, 218 insertions(+), 49 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/75/227175/1
diff --git a/files/logstash/filter-irc-banglog.conf
b/files/logstash/filter-irc-banglog.conf
deleted file mode 100644
index 44cfb5c..0000000
--- a/files/logstash/filter-irc-banglog.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-filter {
- if [type] == "irc" and [message] =~ /^!log / {
- mutate {
- add_tag => [ "es", "banglog" ]
- }
-
- if [channel] == "#wikimedia-labs" {
- grok {
- match => [ "message", "^!log %{NOTSPACE:project} " ]
- named_captures_only => true
- }
- }
-
- if [channel] == "#wikimedia-releng" {
- mutate {
- add_field => [ "project", "qa" ]
- }
- }
-
- if [channel] == "#wikimedia-operations" {
- mutate {
- add_field => [ "project", "production" ]
- }
- }
- }
-}
diff --git a/files/logstash/filter-stashbot-sal.conf
b/files/logstash/filter-stashbot-sal.conf
new file mode 100644
index 0000000..cb0c2d3
--- /dev/null
+++ b/files/logstash/filter-stashbot-sal.conf
@@ -0,0 +1,26 @@
+filter {
+ if [type] == "sal" {
+ if [channel] == "#wikimedia-labs" {
+ grok {
+ match => [ "message", "^!log %{NOTSPACE:project}
%{GREEDYDATA:message}$" ]
+ named_captures_only => true
+ }
+ }
+
+ if [channel] == "#wikimedia-releng" {
+ grok {
+ match => [ "message", "^!log %{GREEDYDATA:message}$" ]
+ named_captures_only => true
+ add_field => [ "project", "releng" ]
+ }
+ }
+
+ if [channel] == "#wikimedia-operations" {
+ grok {
+ match => [ "message", "^!log %{GREEDYDATA:message}$" ]
+ named_captures_only => true
+ add_field => [ "project", "production" ]
+ }
+ }
+ } # end [type] == "sal"
+}
diff --git a/files/logstash/filter-stashbot.conf
b/files/logstash/filter-stashbot.conf
new file mode 100644
index 0000000..1dede3c
--- /dev/null
+++ b/files/logstash/filter-stashbot.conf
@@ -0,0 +1,14 @@
+filter {
+ if [type] == "irc" {
+ mutate {
+ add_tag => [ "es" ]
+ }
+
+ if [message] =~ /^!log / {
+ clone {
+ clones => [ "sal" ]
+ remove_tag => [ "es" ]
+ }
+ } # end [message] =~ /^!log /
+ } # end [type] == "irc"
+}
diff --git a/files/logstash/stashbot-sal-template.json
b/files/logstash/stashbot-sal-template.json
new file mode 100644
index 0000000..40f39e9
--- /dev/null
+++ b/files/logstash/stashbot-sal-template.json
@@ -0,0 +1,40 @@
+{
+ "template" : "sal",
+ "settings" : {
+ "number_of_shards" : 1,
+ "number_of_replicas" : 0,
+ "index.cache.field.type" : "soft",
+ "index.refresh_interval" : "5s",
+ "analysis" : {
+ "analyzer" : {
+ "default" : {
+ "type" : "standard",
+ "stopwords" : "_none_"
+ }
+ }
+ }
+ },
+ "mappings" : {
+ "sal" : {
+ "_all" : {"enabled" : false},
+ "dynamic_templates" : [ {
+ "string_fields" : {
+ "match" : "*",
+ "match_mapping_type" : "string",
+ "mapping" : { "type" : "string", "index" : "analyzed", "omit_norms"
: true }
+ }
+ } ],
+ "properties" : {
+ "@timestamp" : { "type" : "date", "index" : "not_analyzed" },
+ "@version" : { "type" : "string", "index" : "not_analyzed" },
+ "channel" : { "type" : "string", "index" : "not_analyzed" },
+ "message" : { "type" : "string", "index" : "analyzed", "omit_norms" :
true },
+ "nick" : { "type" : "string", "index" : "not_analyzed" }
+ "project" : { "type" : "string", "index" : "not_analyzed" }
+ "server" : { "type" : "string", "index" : "not_analyzed" }
+ "tags" : { "type" : "string", "index" : "not_analyzed", "index_name" :
"tag" },
+ "type" : { "type" : "string", "index" : "not_analyzed" },
+ }
+ }
+ }
+}
diff --git a/files/logstash/stashbot-template.json
b/files/logstash/stashbot-template.json
new file mode 100644
index 0000000..6ed2a02
--- /dev/null
+++ b/files/logstash/stashbot-template.json
@@ -0,0 +1,39 @@
+{
+ "template" : "logstash-*",
+ "settings" : {
+ "number_of_shards" : 1,
+ "number_of_replicas" : 0,
+ "index.cache.field.type" : "soft",
+ "index.refresh_interval" : "5s",
+ "analysis" : {
+ "analyzer" : {
+ "default" : {
+ "type" : "standard",
+ "stopwords" : "_none_"
+ }
+ }
+ }
+ },
+ "mappings" : {
+ "_default_" : {
+ "_all" : {"enabled" : false},
+ "dynamic_templates" : [ {
+ "string_fields" : {
+ "match" : "*",
+ "match_mapping_type" : "string",
+ "mapping" : { "type" : "string", "index" : "analyzed", "omit_norms"
: true }
+ }
+ } ],
+ "properties" : {
+ "@timestamp" : { "type" : "date", "index" : "not_analyzed" },
+ "@version" : { "type" : "string", "index" : "not_analyzed" },
+ "channel" : { "type" : "string", "index" : "not_analyzed" },
+ "message" : { "type" : "string", "index" : "analyzed", "omit_norms" :
true },
+ "nick" : { "type" : "string", "index" : "not_analyzed" }
+ "server" : { "type" : "string", "index" : "not_analyzed" }
+ "tags" : { "type" : "string", "index" : "not_analyzed", "index_name" :
"tag" },
+ "type" : { "type" : "string", "index" : "not_analyzed" },
+ }
+ }
+ }
+}
diff --git a/hieradata/labs/logstash/host/stashbot01.yaml
b/hieradata/labs/logstash/host/stashbot01.yaml
new file mode 100644
index 0000000..2358e27
--- /dev/null
+++ b/hieradata/labs/logstash/host/stashbot01.yaml
@@ -0,0 +1,17 @@
+---
+# Elasticsearch cluster
+elasticsearch::auto_create_index: true
+elasticsearch::cluster_name: stashbot
+elasticsearch::expected_nodes: 1
+elasticsearch::heap_memory: 8G
+elasticsearch::minimum_master_nodes: 1
+elasticsearch::plugins_dir: /srv/deployment/elasticsearch/plugins
+elasticsearch::recover_after_nodes: 1
+elasticsearch::recover_after_time: 1m
+elasticsearch::unicast_hosts:
+ - stashbot01.logstash.eqiad.wmflabs
+elasticsearch::script_disable_dynamic: true
+
+# Logstash tuning
+logstash::filter_workers: 2
+logstash::heap_memory_mb: 512
diff --git a/manifests/role/logstash.pp b/manifests/role/logstash.pp
index b6b30aa..3b85b3c 100644
--- a/manifests/role/logstash.pp
+++ b/manifests/role/logstash.pp
@@ -116,29 +116,6 @@
}
}
-# == Class: role::logstash::ircbot
-#
-# Sets up an IRC Bot to log messages from certain IRC channels
-class role::logstash::ircbot {
- require ::role::logstash
-
- $irc_name = $::logstash_irc_name ? {
- undef => "logstash-${::labsproject}",
- default => $::logstash_irc_name,
- }
-
- logstash::input::irc { 'freenode':
- user => $irc_name,
- nick => $irc_name,
- channels => ['#wikimedia-labs', '#wikimedia-releng',
'#wikimedia-operations'],
- }
-
- logstash::conf { 'filter_irc_banglog':
- source => 'puppet:///files/logstash/filter-irc-banglog.conf',
- priority => 50,
- }
-}
-
# == Class: role::logstash::puppetreports
#
# Set up a TCP listener to listen for puppet failure reports.
@@ -199,3 +176,85 @@
require => File['/etc/logstash/apifeatureusage-template.json'],
}
}
+
+# == Class: role::logstash::stashbot
+#
+# Configure logstash to record IRC channel messages
+#
+# == Parameters:
+# [*irc_user*]
+# IRC username
+#
+# [*irc_pass*]
+# IRC password
+#
+# [*irc_nick*]
+# IRC nick
+#
+# [*irc_real*]
+# IRC real name
+#
+# [*channels*]
+# List of channels to join and log
+#
+class role::logstash::stashbot (
+ $irc_user = 'stashbot',
+ $irc_pass = undef,
+ $irc_nick = 'stashbot',
+ $irc_real = 'Wikimedia Tool Labs IRC bot',
+ $channels = [],
+) {
+ include standard
+ include ::logstash
+ include ::role::logstash::elasticsearch
+
+ logstash::input::irc { 'freenode':
+ user => $irc_user,
+ password => $irc_pass,
+ nick => $irc_name,
+ real => $irc_real,
+ channels => $channels,
+ }
+
+ logstash::conf { 'filter_strip_ansi_color':
+ source => 'puppet:///files/logstash/filter-strip-ansi-color.conf',
+ priority => 15,
+ }
+
+ logstash::conf { 'filter_stashbot':
+ source => 'puppet:///files/logstash/filter-stashbot.conf',
+ priority => 20,
+ }
+
+ logstash::conf { 'filter_stashbot_sal':
+ source => 'puppet:///files/logstash/filter-stashbot-sal.conf',
+ priority => 50,
+ }
+
+ file { '/etc/logstash/stashbot-template.json':
+ ensure => present,
+ source => 'puppet:///files/logstash/stashbot-template.json',
+ }
+ logstash::output::elasticsearch { 'logstash':
+ host => '127.0.0.1',
+ guard_condition => '"es" in [tags]',
+ priority => 90,
+ template => '/etc/logstash/stashbot-template.json',
+ require => File['/etc/logstash/stashbot-template.json'],
+ }
+
+ # Special indexing for SAL messages
+ file { '/etc/logstash/stashbot-sal-template.json':
+ ensure => present,
+ source => 'puppet:///files/logstash/stashbot-sal-template.json',
+ }
+ logstash::output::elasticsearch { 'sal':
+ host => $host,
+ index => 'sal',
+ guard_condition => '[type] == "sal"',
+ priority => 95,
+ template => '/etc/logstash/stashbot-sal-template.json',
+ require => File['/etc/logstash/stashbot-sal-template.json'],
+ }
+}
+
--
To view, visit https://gerrit.wikimedia.org/r/227175
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia04219138f3dab667d7c2a667994681aeeabc563
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
