Filippo Giunchedi has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/316543

Change subject: Introduce mtail module
......................................................................

Introduce mtail module

The idea is to extract metrics from (sys)logs and report the results as time
series to graphite/statsd/prometheus

Bug: T147923
Change-Id: I7fbde97762decbac4cb1ec1984eb987f63ec60e6
---
A modules/mtail/files/programs/kernel.mtail
A modules/mtail/files/programs/linecount.mtail
A modules/mtail/manifests/init.pp
A modules/mtail/manifests/program.pp
A modules/mtail/templates/default.erb
5 files changed, 149 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/43/316543/1

diff --git a/modules/mtail/files/programs/kernel.mtail 
b/modules/mtail/files/programs/kernel.mtail
new file mode 100644
index 0000000..9342ebc
--- /dev/null
+++ b/modules/mtail/files/programs/kernel.mtail
@@ -0,0 +1,57 @@
+# Syslog decorator
+def syslog 
{/(?P<date>(?P<legacy_date>\w+\s+\d+\s+\d+:\d+:\d+)|(?P<rfc3339_date>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+[+-]\d{2}:\d{2}))/
 +
+    
/\s+(?:\w+@)?(?P<hostname>[\w\.-]+)\s+(?P<application>[\w\.-]+)(?:\[(?P<pid>\d+)\])?:\s+(?P<message>.*)/
 {
+        len($legacy_date) > 0 {
+            strptime($2, "Jan _2 15:04:05")
+        }
+        len($rfc3339_date) > 0 {
+            strptime($rfc3339_date, "2006-01-02T03:04:05-0700")
+        }
+        next
+    }
+}
+
+@syslog {
+counter mce_events
+counter cpu_throttled
+counter edac_correctable_errors
+counter edac_uncorrectable_errors
+counter edac_events
+
+counter upstart_respawn by service
+counter oom_kill by binary
+counter segfault by binary
+
+/kernel: \[[\d.]+\] mce: .*Machine check events logged/ {
+  mce_events++
+}
+
+/kernel: \[[\d.]+\] .* above threshold, cpu clock throttled/ {
+  cpu_throttled++
+}
+
+/kernel: \[[\d.]+\] EDAC / {
+  edac_events++
+}
+
+/kernel: \[[\d.]+\] EDAC MC\d+: (\d+) CE/ {
+  edac_correctable_errors += $1
+}
+
+/kernel: \[[\d.]+\] EDAC MC\d+: (\d+) UE/ {
+  edac_uncorrectable_errors += $1
+}
+
+/kernel: \[[\d.]+\] init: (?P<service>[\S]+) .*process ended, respawning/ {
+  upstart_respawn[$service]++
+}
+
+/kernel: \[[\d.]+\] (?P<binary>[\S]+) invoked oom-killer:.*/ {
+  oom_kill[$binary]++
+}
+
+/kernel: \[[\d.]+\] (?P<binary>[^\[]+)\[\d+\]: segfault at.*/ {
+  segfault[$binary]++
+}
+
+}
diff --git a/modules/mtail/files/programs/linecount.mtail 
b/modules/mtail/files/programs/linecount.mtail
new file mode 100644
index 0000000..fcb84a9
--- /dev/null
+++ b/modules/mtail/files/programs/linecount.mtail
@@ -0,0 +1,8 @@
+# Copyright 2011 Google Inc. All Rights Reserved.
+# This file is available under the Apache license.
+
+counter line_count
+
+/$/ {
+  line_count++
+}
diff --git a/modules/mtail/manifests/init.pp b/modules/mtail/manifests/init.pp
new file mode 100644
index 0000000..cacedcb
--- /dev/null
+++ b/modules/mtail/manifests/init.pp
@@ -0,0 +1,45 @@
+# == Class: mtail
+#
+# Setup mtail to scan $logs and report metrics based on programs in /etc/mtail.
+#
+# === Parameters
+#
+# [*logs*]
+#   Array of log files to follow
+#
+# [*port*]
+#   TCP port to listen to for Prometheus-style metrics
+#
+# [*graphite_hostport*]
+#   Also send metrics via graphite line-oriented protocol to this host:port.
+#
+# [*enabled*]
+#   Whether to start mtail at boot
+
+class mtail (
+  $logs = ['/var/log/syslog'],
+  $port = '3903',
+  $graphite_hostport = 'graphite-in.eqiad.wmnet:2003',
+  $enabled = '1',
+) {
+    validate_array($logs)
+    validate_re($port, '^[0-9]+$')
+    validate_string($graphite_hostport)
+    validate_string($enabled)
+
+    require_package('mtail')
+
+    file { '/etc/default/mtail':
+        ensure  => present,
+        owner   => 'root',
+        group   => 'root',
+        mode    => '0444',
+        content => template('mtail/default.erb'),
+        notify  => Service['mtail'],
+    }
+
+    service { 'mtail':
+        ensure => running,
+        before => File['/etc/default/mtail'],
+    }
+}
diff --git a/modules/mtail/manifests/program.pp 
b/modules/mtail/manifests/program.pp
new file mode 100644
index 0000000..e54fd74
--- /dev/null
+++ b/modules/mtail/manifests/program.pp
@@ -0,0 +1,35 @@
+# == Define: mtail::program
+#
+# Install an mtail "program" to extract metrics from log files.
+#
+# === Parameters
+#
+# [*ensure*]
+#   The usual metaparameter.
+#
+# [*content*]
+#   The content of the file provided as a string. Either this or
+#   'source' must be specified.
+#
+# [*source*]
+#   The content of the file provided as a puppet:/// file reference.
+#   Either this or 'content' must be specified.
+#
+define mtail::program(
+    $ensure   = present,
+    $content  = undef,
+    $source   = undef,
+) {
+    validate_ensure($ensure)
+
+    include ::mtail
+
+    $basename = regsubst($title, '\W', '-', 'G')
+    $filename = "/etc/mtail/${basename}.mtail"
+
+    file { $filename:
+        ensure  => $ensure,
+        content => $content,
+        source  => $source,
+    }
+}
diff --git a/modules/mtail/templates/default.erb 
b/modules/mtail/templates/default.erb
new file mode 100644
index 0000000..fe5a9ed
--- /dev/null
+++ b/modules/mtail/templates/default.erb
@@ -0,0 +1,4 @@
+ENABLED=<%= @enabled %>
+GRAPHITE_HOSTPORT=<%= @graphite_hostport %>
+PORT=<%= @port %>
+LOGS=<%= @logs.join(',') %>

-- 
To view, visit https://gerrit.wikimedia.org/r/316543
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I7fbde97762decbac4cb1ec1984eb987f63ec60e6
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Filippo Giunchedi <fgiunch...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to