Gehel has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/344964 )
Change subject: Upgrade logstash to 5.x
......................................................................
Upgrade logstash to 5.x
Logstash 5.x is compatible with elasticsearch 2.x through 5.x, whereas
the currently deployed version of logstash is not compatible with
elasticsearch 5.x. This should roll out prior to upgrading elasticsearch
in the logging cluster to 5.x.
* JVM config is now set via /etc/logstash/jvm.options, rather than
/etc/default/logstash. Provided config is the same as upstream with
one update to specify our custom heap sizes.
* Depends on updated operations/software/logstash/plugins repo to
provide a 'plugin pack'.
* The logstash/plugins trebuchet repository is now git-fat enabled to
allow deployment of the new plugin pack.
* renamed logstash::heap_memory_mb to logstash::heap_memory to be
consistent with other java modules.
* renamed logstash::filter_works to logstash::pipeline_works to match
upstream rename
* the kafka input plugin now takes a list of kafka brokers, rather than
a zookeeper host, to bootstrap itself with.
* Made all the logstash config files 0444 instead of 0440. There is
nothing secret in them and it makes debugging more of a pain by
requiring sudo.
Bug: T154473
Change-Id: Ica36805c6fb09a33e9cae6788711d4ed3d7beb9b
---
M hieradata/common/role/deployment.yaml
M hieradata/labs/deployment-prep/host/deployment-logstash2.yaml
M hieradata/role/common/logstash/collector.yaml
M modules/logstash/manifests/conf.pp
M modules/logstash/manifests/init.pp
M modules/logstash/manifests/input/kafka.pp
D modules/logstash/templates/default.erb
M modules/logstash/templates/input/kafka.erb
A modules/logstash/templates/jvm.options.erb
D modules/logstash/templates/logstash.service.erb
M modules/logstash/templates/output/elasticsearch.erb
M modules/role/manifests/logstash/eventlogging.pp
12 files changed, 142 insertions(+), 86 deletions(-)
Approvals:
jenkins-bot: Verified
Gehel: Looks good to me, approved
diff --git a/hieradata/common/role/deployment.yaml
b/hieradata/common/role/deployment.yaml
index b642364..7612021 100644
--- a/hieradata/common/role/deployment.yaml
+++ b/hieradata/common/role/deployment.yaml
@@ -41,6 +41,7 @@
gitfat_enabled: true
upstream:
https://gerrit.wikimedia.org/r/operations/software/logstash-logback-encoder
logstash/plugins:
+ gitfat_enabled: true
upstream:
https://gerrit.wikimedia.org/r/operations/software/logstash/plugins
wikimedia/discovery/analytics:
upstream: https://gerrit.wikimedia.org/r/wikimedia/discovery/analytics
diff --git a/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml
b/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml
index 14653eb..1564388 100644
--- a/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml
+++ b/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml
@@ -15,9 +15,6 @@
elasticsearch::auto_create_index: '+logstash-*,-*'
# Logstash specific
-logstash::filter_workers: 1
-logstash::heap_memory_mb: 128
-
logstash::cluster_hosts:
- deployment-logstash2.deployment-prep.eqiad.wmflabs
diff --git a/hieradata/role/common/logstash/collector.yaml
b/hieradata/role/common/logstash/collector.yaml
index 6c9bb53..52945eb 100644
--- a/hieradata/role/common/logstash/collector.yaml
+++ b/hieradata/role/common/logstash/collector.yaml
@@ -28,8 +28,8 @@
# TODO: the multiline filter that is used in several places in the
# current configuration isn't thread safe and can cause crashes or
# garbled output when used with more than one thread worker.
-logstash::filter_workers: 1
-logstash::heap_memory_mb: 256
+logstash::pipeline_workers: 1
+logstash::heap_memory: 256m
logstash::cluster_hosts:
@@ -57,3 +57,4 @@
debdeploy::grains:
debdeploy-logstash:
value: standard
+
diff --git a/modules/logstash/manifests/conf.pp
b/modules/logstash/manifests/conf.pp
index 72bd1f5..75332ea 100644
--- a/modules/logstash/manifests/conf.pp
+++ b/modules/logstash/manifests/conf.pp
@@ -36,7 +36,7 @@
source => $source,
owner => 'logstash',
group => 'logstash',
- mode => '0440',
+ mode => '0444',
notify => Service['logstash'],
}
}
diff --git a/modules/logstash/manifests/init.pp
b/modules/logstash/manifests/init.pp
index 7a2a2da..d84d6b3 100644
--- a/modules/logstash/manifests/init.pp
+++ b/modules/logstash/manifests/init.pp
@@ -5,20 +5,20 @@
# pipelines that ingress log data from various sources in a variety of formats.
#
# == Parameters:
-# - $heap_memory_mb: amount of memory to allocate to logstash in megabytes.
-# - $filter_workers: number of worker threads to run to process filters
+# - $heap_memory: amount of memory to allocate to logstash.
+# - $pipeline_workers: number of worker threads to run to process filters
#
# == Sample usage:
#
# class { 'logstash':
-# heap_memory_mb => 128,
-# filter_workers => 3,
+# heap_memory => "192m",
+# pipeline_workers => 3,
# }
#
class logstash(
- $heap_memory_mb = 64,
- $filter_workers = 1,
- $java_package = 'openjdk-8-jdk',
+ $heap_memory = '192m',
+ $pipeline_workers = 1,
+ $java_package = 'openjdk-8-jdk',
) {
require_package($java_package)
@@ -28,11 +28,39 @@
}
package { 'logstash/plugins':
- provider => 'trebuchet',
+ provider => 'trebuchet',
+ }
+
+ $plugin_zip_path =
'/srv/deployment/logstash/plugins/target/releases/plugins-latest.zip'
+ exec { 'install-logstash-plugins':
+ command => "/usr/share/logstash/bin/logstash-plugin install
file://${plugin_zip_path} && /usr/bin/sha256sum ${plugin_zip_path} >
/etc/logstash/plugins.sha256sum",
+ # Only install plugins if hash of latest does not match stored state
+ unless => "/usr/bin/test \"$(/bin/cat
/etc/logstash/plugins.sha256sum)\" = \"$(/usr/bin/sha256sum
${plugin_zip_path})\"",
+ # Intentionally does not notify Service['logstash'], preferring a
manual rolling restart of logstash servers
+ require => Package['logstash'],
+ before => Service['logstash'],
}
file { '/etc/default/logstash':
- content => template('logstash/default.erb'),
+ ensure => absent,
+ }
+
+ file { '/etc/logstash/jvm.options':
+ content => template('logstash/jvm.options.erb'),
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ require => Package['logstash'],
+ notify => Service['logstash'],
+ }
+
+ file { '/etc/logstash/logstash.yml':
+ content => ordered_yaml({
+ 'path.data' => '/var/lib/logstash',
+ 'path.config' => '/etc/logstash/conf.d',
+ 'path.logs' => '/var/log/logstash',
+ 'pipeline.workers' => $pipeline_workers,
+ }),
owner => 'root',
group => 'root',
mode => '0444',
@@ -51,9 +79,11 @@
require => Package['logstash'],
}
+ # Older 1.x versions of logstash needed this file deployed,
+ # but 5.x comes with a sensible service definition for systemd
+ # in /etc/systemd/logstash.service
file { '/lib/systemd/system/logstash.service':
- content => template('logstash/logstash.service.erb'),
- notify => Service['logstash'],
+ ensure => absent,
}
service { 'logstash':
diff --git a/modules/logstash/manifests/input/kafka.pp
b/modules/logstash/manifests/input/kafka.pp
index dec9963..dad807b 100644
--- a/modules/logstash/manifests/input/kafka.pp
+++ b/modules/logstash/manifests/input/kafka.pp
@@ -11,23 +11,23 @@
# - $tags: Array of tags to be added to the logs. Default: [$title].
# - $topic: Kafka topic. Default: $title.
# - $type: Log type to be passed to Logstash. Default: 'kafka'.
-# - $zk_connect: Zookeeper host and port (and optionally: chroot path).
-# Format: 'some.zookeeper.host:1234/chroot/path'. For more info, see:
+# - $bootstrap_servers: Kafka servers to boostrap from. This list should be
+# a string in the form of `host1:port1,host2:port2. For more info, see:
#
https://www.elastic.co/guide/en/logstash/current/plugins-inputs-kafka.html
#
# == Sample usage:
#
# logstash::input::kafka { 'some_topic':
-# zk_connect => 'some.zookeeper.host:1234/chroot/path',
+# boostrap_servers => 'kafka1001:9092,kafka1002:9092',
# }
#
define logstash::input::kafka(
- $ensure = present,
- $priority = 10,
- $tags = [$title],
- $topic = $title,
- $type = 'kafka',
- $zk_connect = '',
+ $bootstrap_servers,
+ $ensure = present,
+ $priority = 10,
+ $tags = [$title],
+ $topic = $title,
+ $type = 'kafka',
) {
logstash::conf { "input-kafka-${title}":
ensure => $ensure,
diff --git a/modules/logstash/templates/default.erb
b/modules/logstash/templates/default.erb
deleted file mode 100644
index 8d5c575..0000000
--- a/modules/logstash/templates/default.erb
+++ /dev/null
@@ -1,21 +0,0 @@
-#####################################################################
-## THIS FILE IS MANAGED BY PUPPET
-## puppet:///logstash/default.erb
-#####################################################################
-
-# Start logstash on boot?
-START=yes
-
-# Location of the logstash jar file
-LS_JAR=/opt/logstash/logstash.jar
-
-# Java JVM options
-LS_JAVA_OPTS="-Xms<%= @heap_memory_mb %>m -Xmx<%= @heap_memory_mb %>m
-Djava.io.tmpdir=/var/lib/logstash"
-
-# Location of configuration files.
-# All files in this directory are concatenated in alphabetical order.
-CONF_DIR=/etc/logstash/conf.d
-
-# Logstash options
-# Note: the sysv init script prepends these options with "agent -f ${CONF_DIR}"
-LS_OPTS="--log /var/log/logstash/logstash.log --filterworkers <%=
@filter_workers %> --pluginpath /srv/deployment/logstash/plugins"
diff --git a/modules/logstash/templates/input/kafka.erb
b/modules/logstash/templates/input/kafka.erb
index cebef84..2f8eb1a 100644
--- a/modules/logstash/templates/input/kafka.erb
+++ b/modules/logstash/templates/input/kafka.erb
@@ -3,10 +3,10 @@
# Some options have not been added, if you want to add more options, see:
#
https://www.elastic.co/guide/en/logstash/current/plugins-inputs-kafka.html
<%- if @tags -%>
- tags => ["<%= @tags.join('", "') %>"]
+ tags => ["<%= @tags.join('", "') %>"]
<%- end -%>
- topic_id => "<%= @topic %>"
- type => "<%= @type %>"
- zk_connect => "<%= @zk_connect %>"
+ topics => ["<%= @topic %>"]
+ type => "<%= @type %>"
+ bootstrap_servers => "<%= @bootstrap_servers %>"
}
}
diff --git a/modules/logstash/templates/jvm.options.erb
b/modules/logstash/templates/jvm.options.erb
new file mode 100644
index 0000000..d242b00
--- /dev/null
+++ b/modules/logstash/templates/jvm.options.erb
@@ -0,0 +1,79 @@
+#####################################################################
+## THSI FILE IS MANAGED BY PUPPET
+## puppet://modules/logstash/jvm.options.erb
+#####################################################################
+
+## JVM configuration
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms<%= @heap_memory %>
+-Xmx<%= @heap_memory %>
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseParNewGC
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# disable calls to System#gc
+-XX:+DisableExplicitGC
+
+## locale
+# Set the locale language
+#-Duser.language=en
+
+# Set the locale country
+#-Duser.country=US
+
+# Set the locale variant, if any
+#-Duser.variant=
+
+## basic
+
+# set the I/O temp directory
+#-Djava.io.tmpdir=$HOME
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+#-Djna.nosys=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof
+
+## GC logging
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${LS_GC_LOG_FILE}
diff --git a/modules/logstash/templates/logstash.service.erb
b/modules/logstash/templates/logstash.service.erb
deleted file mode 100644
index 8e8f470..0000000
--- a/modules/logstash/templates/logstash.service.erb
+++ /dev/null
@@ -1,29 +0,0 @@
-[Unit]
-Description=Logstash
-Requires=elasticsearch.service
-
-[Install]
-WantedBy=multi-user.target
-
-[Service]
-User=logstash
-Group=logstash
-WorkingDirectory=/opt/logstash
-Restart=always
-Type=simple
-PIDFile=/var/run/logstash.pid
-LimitNOFILE=16384
-EnvironmentFile=/etc/default/logstash
-Environment="LS_USER=logstash"
-Environment="LS_GROUP=logstash"
-Environment="LS_HOME="
-Environment="LS_HEAP_SIZE=500m"
-Environment="LS_JAVA_OPTS=-Djava.io.tmpdir=/var/lib/logstash"
-Environment="LS_LOG_DIR=/var/log/logstash"
-Environment="LS_LOG_FILE=/var/log/logstash/logstash.log"
-Environment="LS_CONF_DIR=/etc/logstash/conf.d"
-Environment="LS_OPEN_FILES=16384"
-Environment="LS_NICE=19"
-ExecStart=/opt/logstash/bin/logstash -f /etc/logstash/conf.d -l
/var/log/logstash/logstash.log --log /var/log/logstash/logstash.log
--filterworkers 1 --pluginpath /srv/deployment/logstash/plugins
-StandardOutput=journal
-StandardError=journal
diff --git a/modules/logstash/templates/output/elasticsearch.erb
b/modules/logstash/templates/output/elasticsearch.erb
index 5f02711..c26f825 100644
--- a/modules/logstash/templates/output/elasticsearch.erb
+++ b/modules/logstash/templates/output/elasticsearch.erb
@@ -4,11 +4,9 @@
# be tightly coupled by the binary protocol
elasticsearch {
flush_size => <%= @flush_size %>
- host => "<%= @host %>"
+ hosts => ["<%= @host %>:<%= @port %>"]
idle_flush_time => <%= @idle_flush_time %>
index => "<%= @index %>"
- port => <%= @port %>
- protocol => "http"
<% if @template %>
manage_template => true
template => "<%= @template %>"
diff --git a/modules/role/manifests/logstash/eventlogging.pp
b/modules/role/manifests/logstash/eventlogging.pp
index 04177a1..7cc5cc6 100644
--- a/modules/role/manifests/logstash/eventlogging.pp
+++ b/modules/role/manifests/logstash/eventlogging.pp
@@ -11,9 +11,9 @@
$kafka_config = kafka_config('analytics')
logstash::input::kafka { $topic:
- tags => [$topic, 'kafka'],
- type => 'eventlogging',
- zk_connect => $kafka_config['zookeeper']['url'],
+ tags => [$topic, 'kafka'],
+ type => 'eventlogging',
+ bootstrap_servers => $kafka_config['brokers']['string'],
}
# lint:ignore:puppet_url_without_modules
logstash::conf { 'filter_eventlogging':
--
To view, visit https://gerrit.wikimedia.org/r/344964
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ica36805c6fb09a33e9cae6788711d4ed3d7beb9b
Gerrit-PatchSet: 15
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: EBernhardson <[email protected]>
Gerrit-Reviewer: EBernhardson <[email protected]>
Gerrit-Reviewer: Gehel <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
