Gehel has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/395970 )
Change subject: elasticsearch: use the canonical definition of logstash host
......................................................................
elasticsearch: use the canonical definition of logstash host
Bug: T182304
Change-Id: I7f0689a6f808f6a17849303b81900abf1520b834
---
M hieradata/common.yaml
M hieradata/labs/deployment-prep/common.yaml
M hieradata/role/codfw/elasticsearch/cirrus.yaml
M hieradata/role/eqiad/elasticsearch/cirrus.yaml
M hieradata/role/eqiad/elasticsearch/relforge.yaml
M modules/elasticsearch/manifests/init.pp
M modules/elasticsearch/spec/classes/ship_logs_to_logstash_rspec.rb
M modules/elasticsearch/templates/log4j2.properties.erb
M modules/elasticsearch/templates/logging.yml.erb
M modules/profile/manifests/elasticsearch.pp
10 files changed, 18 insertions(+), 30 deletions(-)
Approvals:
jenkins-bot: Verified
Gehel: Looks good to me, approved
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 4b8594c..099761c 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -517,6 +517,7 @@
# Logging: logstash, udp2log
logstash_host: "logstash.svc.eqiad.wmnet"
logstash_syslog_port: 10514
+logstash_gelf_port: 12201
udp2log_aggregator: "udplog:8420"
tcpircbot_host: 'icinga.wikimedia.org'
diff --git a/hieradata/labs/deployment-prep/common.yaml
b/hieradata/labs/deployment-prep/common.yaml
index f0f3209..3c3fbe2 100644
--- a/hieradata/labs/deployment-prep/common.yaml
+++ b/hieradata/labs/deployment-prep/common.yaml
@@ -144,6 +144,7 @@
"udp2log_aggregator": deployment-fluorine02.deployment-prep.eqiad.wmflabs:8420
"logstash_host": deployment-logstash2.deployment-prep.eqiad.wmflabs
"logstash_syslog_port": 10514
+"logstash_gelf_port": 12201
mediawiki_memcached_servers:
- 10.68.23.25:11211:1 # deployment-memc04
- 10.68.23.49:11211:1 # deployment-memc05
@@ -230,8 +231,6 @@
- deployment-elastic06
- deployment-elastic07
"profile::elasticsearch::cluster_name": beta-search
-"profile::elasticsearch::graylog_hosts":
- - deployment-logstash2.deployment-prep.eqiad.wmflabs
"profile::elasticsearch::reindex_remote_whitelist": ''
profile::elasticsearch::search_shard_count_limit: 1000
# By default elasticsearch sets this to ((# of available_processors * 3) / 2)
+ 1,
diff --git a/hieradata/role/codfw/elasticsearch/cirrus.yaml
b/hieradata/role/codfw/elasticsearch/cirrus.yaml
index ed089c7..7bbd947 100644
--- a/hieradata/role/codfw/elasticsearch/cirrus.yaml
+++ b/hieradata/role/codfw/elasticsearch/cirrus.yaml
@@ -53,9 +53,6 @@
# We should be able to run "OK" with 11 servers.
profile::elasticsearch::recover_after_nodes: 11
-profile::elasticsearch::graylog_hosts:
- - logstash.svc.eqiad.wmnet
-
profile::elasticsearch::certificate_name: 'search.svc.codfw.wmnet'
profile::base::dns_alt_names: 'search.svc.codfw.wmnet'
diff --git a/hieradata/role/eqiad/elasticsearch/cirrus.yaml
b/hieradata/role/eqiad/elasticsearch/cirrus.yaml
index 7592320..eee5704 100644
--- a/hieradata/role/eqiad/elasticsearch/cirrus.yaml
+++ b/hieradata/role/eqiad/elasticsearch/cirrus.yaml
@@ -54,9 +54,6 @@
# We should be able to run "OK" with 24 servers.
profile::elasticsearch::recover_after_nodes: 24
-profile::elasticsearch::graylog_hosts:
- - logstash.svc.eqiad.wmnet
-
profile::elasticsearch::certificate_name: 'search.svc.eqiad.wmnet'
profile::base::dns_alt_names: 'search.svc.eqiad.wmnet'
diff --git a/hieradata/role/eqiad/elasticsearch/relforge.yaml
b/hieradata/role/eqiad/elasticsearch/relforge.yaml
index 39e498d..4ef87a4 100644
--- a/hieradata/role/eqiad/elasticsearch/relforge.yaml
+++ b/hieradata/role/eqiad/elasticsearch/relforge.yaml
@@ -13,9 +13,6 @@
profile::elasticsearch::heap_memory: '30G'
profile::elasticsearch::minimum_master_nodes: 1
-profile::elasticsearch::graylog_hosts:
- - logstash.svc.eqiad.wmnet
-
profile::elasticsearch::certificate_name: 'relforge.svc.eqiad.wmnet'
profile::base::dns_alt_names: 'relforge.svc.eqiad.wmnet'
diff --git a/modules/elasticsearch/manifests/init.pp
b/modules/elasticsearch/manifests/init.pp
index 077c2ad..e711aa1 100644
--- a/modules/elasticsearch/manifests/init.pp
+++ b/modules/elasticsearch/manifests/init.pp
@@ -111,8 +111,8 @@
$bulk_thread_pool_capacity = undef,
$search_thread_pool_executors = undef,
$load_fixed_bitset_filters_eagerly = true,
- $graylog_hosts = undef,
- $graylog_port = 12201,
+ $logstash_host = undef,
+ $logstash_gelf_port = 12201,
$gc_log = true,
$java_package = 'openjdk-8-jdk',
$version = 5,
@@ -138,15 +138,10 @@
fail('script_max_compilations_per_minute should be > 0')
}
- # if no graylog_host is given, do not send logs
- $send_logs_to_logstash = is_array($graylog_hosts)
+ $send_logs_to_logstash = $logstash_host != undef
- if $send_logs_to_logstash {
- validate_array($graylog_hosts)
- $rotated_graylog_host = fqdn_rotate($graylog_hosts, $::hostname)
- $graylog_host = $rotated_graylog_host[0]
- validate_string($graylog_host)
- # validate_integer($graylog_port) // should be uncommented when we
upgrade to stdlib 4.x
+ if $logstash_host {
+ validate_string($logstash_host)
}
$gc_log_flags = $gc_log ? {
diff --git a/modules/elasticsearch/spec/classes/ship_logs_to_logstash_rspec.rb
b/modules/elasticsearch/spec/classes/ship_logs_to_logstash_rspec.rb
index fd700f5..9cf4838 100644
--- a/modules/elasticsearch/spec/classes/ship_logs_to_logstash_rspec.rb
+++ b/modules/elasticsearch/spec/classes/ship_logs_to_logstash_rspec.rb
@@ -14,7 +14,7 @@
describe 'when sending logs to logstash' do
let(:params) { { :cluster_name => 'my_cluster_name',
- :graylog_hosts => [ 'logstash.example.net', 'other' ],
+ :logstash_host => 'logstash.example.net',
:publish_host => '127.0.0.1',
} }
let(:facts) { { :lsbdistrelease => 'ubuntu',
diff --git a/modules/elasticsearch/templates/log4j2.properties.erb
b/modules/elasticsearch/templates/log4j2.properties.erb
index a0be0b7..e079d12 100644
--- a/modules/elasticsearch/templates/log4j2.properties.erb
+++ b/modules/elasticsearch/templates/log4j2.properties.erb
@@ -64,12 +64,12 @@
appender.index_indexing_slow_log_file.layout.type = PatternLayout
appender.index_indexing_slow_log_file.layout.pattern =
[%d{ISO8601}][%-5p][%-25c] %m%n
-<% if @send_logs_to_logstash %>
+<% if @logstash_host %>
# ship_to_logstash needs to also be added to rootLogger to actually ship logs
appender.ship_to_logstash.name = ship_to_logstash
appender.ship_to_logstash.type = Gelf
-appender.ship_to_logstash.host = udp:<%= @graylog_host %>
-appender.ship_to_logstash.port = <%= @graylog_port %>
+appender.ship_to_logstash.host = udp:<%= @logstash_host %>
+appender.ship_to_logstash.port = <%= @logstash_gelf_port %>
appender.ship_to_logstash.originHost = <%= @hostname %>
appender.ship_to_logstash.facility = elasticsearch
appender.ship_to_logstash.extractStackTrace = true
diff --git a/modules/elasticsearch/templates/logging.yml.erb
b/modules/elasticsearch/templates/logging.yml.erb
index 8a1d5bb..4279708 100644
--- a/modules/elasticsearch/templates/logging.yml.erb
+++ b/modules/elasticsearch/templates/logging.yml.erb
@@ -1,4 +1,4 @@
-rootLogger: INFO, file<% if @send_logs_to_logstash %>, ship_to_logstash<% end
%>
+rootLogger: INFO, file<% if @logstash_host %>, ship_to_logstash<% end %>
logger:
# log action execution errors for easier debugging
action: DEBUG
@@ -63,8 +63,8 @@
# ship_to_logstash needs to also be added to rootLogger to actually ship logs
ship_to_logstash:
type: biz.paluch.logging.gelf.log4j.GelfLogAppender
- host: "udp:<%= @graylog_host %>"
- port: <%= @graylog_port %>
+ host: "udp:<%= @logstash_host %>"
+ port: <%= @logstash_gelf_port %>
originHost: <%= @hostname %>
facility: elasticsearch
extractStackTrace: true
diff --git a/modules/profile/manifests/elasticsearch.pp
b/modules/profile/manifests/elasticsearch.pp
index 1b816c0..aa873c8 100644
--- a/modules/profile/manifests/elasticsearch.pp
+++ b/modules/profile/manifests/elasticsearch.pp
@@ -20,7 +20,8 @@
$minimum_master_nodes =
hiera('profile::elasticsearch::minimum_master_nodes'),
$heap_memory = hiera('profile::elasticsearch::heap_memory'),
$expected_nodes = hiera('profile::elasticsearch::expected_nodes'),
- $graylog_hosts = hiera('profile::elasticsearch::graylog_hosts'),
+ $logstash_host = hiera('logstash_host'),
+ $logstash_port = hiera('logstash_gelf_port'),
$rack = hiera('profile::elasticsearch::rack'),
$row = hiera('profile::elasticsearch::row'),
$awareness_attributes =
hiera('profile::elasticsearch::awareness_attributes'),
@@ -105,7 +106,8 @@
heap_memory => $heap_memory,
expected_nodes => $expected_nodes,
master_eligible => $master_eligible,
- graylog_hosts => $graylog_hosts,
+ logstash_host => $logstash_host,
+ logstash_gelf_port => $logstash_port,
version => 5,
search_shard_count_limit => $search_shard_count_limit,
reindex_remote_whitelist => $reindex_remote_whitelist,
--
To view, visit https://gerrit.wikimedia.org/r/395970
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I7f0689a6f808f6a17849303b81900abf1520b834
Gerrit-PatchSet: 5
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Gehel <[email protected]>
Gerrit-Reviewer: DCausse <[email protected]>
Gerrit-Reviewer: Gehel <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Volans <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
