Gehel has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/399652 )
Change subject: thumbor: use the canonical definition of logstash host
..
thumbor: use the canonical definition of logstash host
Also extracts a profile::thumbor class to respect our puppet guidelines
at least on new hiera parameters.
Bug: T182304
Change-Id: I87bb508d633b6f39ead3e6075641db608bb9bbf4
---
M hieradata/common.yaml
M hieradata/labs/deployment-prep/common.yaml
M hieradata/role/codfw/thumbor/mediawiki.yaml
M hieradata/role/eqiad/thumbor/mediawiki.yaml
A modules/profile/manifests/thumbor.pp
M modules/role/manifests/thumbor/mediawiki.pp
6 files changed, 44 insertions(+), 34 deletions(-)
Approvals:
jenkins-bot: Verified
Filippo Giunchedi: Looks good to me, but someone else must approve
Gehel: Looks good to me, approved
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 2cfc1b1..71a9a87 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -522,7 +522,10 @@
logstash_host: "logstash.svc.eqiad.wmnet"
logstash_syslog_port: 10514
logstash_gelf_port: 12201
+# TCP json_lines input
logstash_json_lines_port: 11514
+# UDP logback/json input
+logstash_logback_port: 11514
udp2log_aggregator: "udplog:8420"
tcpircbot_host: 'icinga.wikimedia.org'
diff --git a/hieradata/labs/deployment-prep/common.yaml
b/hieradata/labs/deployment-prep/common.yaml
index f2c54ce..5bad6f5 100644
--- a/hieradata/labs/deployment-prep/common.yaml
+++ b/hieradata/labs/deployment-prep/common.yaml
@@ -140,10 +140,15 @@
default_instance_params:
rpc_address: '%{::ipaddress}'
# end cassandra
-"udp2log_aggregator": deployment-fluorine02.deployment-prep.eqiad.wmflabs:8420
-"logstash_host": deployment-logstash2.deployment-prep.eqiad.wmflabs
-"logstash_syslog_port": 10514
-"logstash_gelf_port": 12201
+udp2log_aggregator: deployment-fluorine02.deployment-prep.eqiad.wmflabs:8420
+logstash_host: deployment-logstash2.deployment-prep.eqiad.wmflabs
+logstash_syslog_port: 10514
+logstash_gelf_port: 12201
+# TCP json_lines input
+logstash_json_lines_port: 11514
+# UDP logback/json input
+logstash_logback_port: 11514
+
mediawiki_memcached_servers:
- 10.68.23.25:11211:1 # deployment-memc04
- 10.68.23.49:11211:1 # deployment-memc05
@@ -357,9 +362,6 @@
recentchange:
topics: ["%{::site}.mediawiki.recentchange"]
profile::eventstreams::rdkafka_config: {}
-
-thumbor::logstash_host: 'deployment-logstash2.deployment-prep.eqiad.wmflabs'
-thumbor::logstash_port: 11514
cache::be_transient_gb: 0
cache::fe_transient_gb: 0
diff --git a/hieradata/role/codfw/thumbor/mediawiki.yaml
b/hieradata/role/codfw/thumbor/mediawiki.yaml
index 9dd4e02..634df35 100644
--- a/hieradata/role/codfw/thumbor/mediawiki.yaml
+++ b/hieradata/role/codfw/thumbor/mediawiki.yaml
@@ -17,7 +17,4 @@
thumbor::poolcounter_server: 'poolcounter2001.codfw.wmnet'
-thumbor::logstash_host: 'logstash.svc.eqiad.wmnet'
-thumbor::logstash_port: 11514
-
thumbor::stl_support: true
diff --git a/hieradata/role/eqiad/thumbor/mediawiki.yaml
b/hieradata/role/eqiad/thumbor/mediawiki.yaml
index 12ac392..4163d6c 100644
--- a/hieradata/role/eqiad/thumbor/mediawiki.yaml
+++ b/hieradata/role/eqiad/thumbor/mediawiki.yaml
@@ -17,7 +17,4 @@
thumbor::poolcounter_server: 'poolcounter1001.eqiad.wmnet'
-thumbor::logstash_host: 'logstash.svc.eqiad.wmnet'
-thumbor::logstash_port: 11514
-
thumbor::stl_support: true
diff --git a/modules/profile/manifests/thumbor.pp
b/modules/profile/manifests/thumbor.pp
new file mode 100644
index 000..c31fa2e
--- /dev/null
+++ b/modules/profile/manifests/thumbor.pp
@@ -0,0 +1,31 @@
+class profile::thumbor(
+$memcached_servers_nutcracker =
hiera('thumbor_memcached_servers_nutcracker'),
+$logstash_host = hiera('logstash_host'),
+$logstash_port = hiera('logstash_logback_port'),
+$swift_sharded_containers =
hiera_array('swift::proxy::shard_container_list'),
+) {
+
+class { '::thumbor::nutcracker':
+thumbor_memcached_servers => $memcached_servers_nutcracker,
+}
+
+class { '::thumbor':
+logstash_host => $logstash_host,
+logstash_port => $logstash_port,
+}
+
+include ::swift::params
+$swift_account_keys = $::swift::params::account_keys
+
+class { '::thumbor::swift':
+swift_key=> $swift_account_keys['mw_thumbor'],
+swift_sharded_containers => $swift_sharded_containers,
+}
+
+ferm::service { 'thumbor':
+proto => 'tcp',
+port => '8800',
+srange => '$DOMAIN_NETWORKS',
+}
+
+}
\ No newline at end of file
diff --git a/modules/role/manifests/thumbor/mediawiki.pp
b/modules/role/manifests/thumbor/mediawiki.pp
index 6214b4c..88f5943 100644
--- a/modules/role/manifests/thumbor/mediawiki.pp
+++ b/modules/role/manifests/thumbor/mediawiki.pp
@@ -10,28 +10,8 @@
include ::mediawiki::packages::fonts
include
