Giuseppe Lavagetto has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/357863 )
Change subject: role::lvs::balancer: refactor to role/profile (step 2)
......................................................................
role::lvs::balancer: refactor to role/profile (step 2)
* Added profile::lvs to manage most part of a loadbalancer setup
* Removed logic from class lvs::balancer, renamed lvs::kernel_config
* Removed lengthy, redundant and error-prone logic to extract service
ips in role::lvs::balancer in favour of a template
* Disentangled pybal setup from lvs setup
The only (small) downside I see in this change is that it will duplicate
some templating logic, but it's honestly better than the verbose and
error-prone list we had before.
Change-Id: I14581d7743abc09b61fed8a5f2cf7b95ea4ed440
---
M modules/lvs/manifests/balancer/runcommand.pp
R modules/lvs/manifests/kernel_config.pp
A modules/profile/manifests/lvs.pp
M modules/profile/manifests/pybal.pp
A modules/profile/templates/lvs/service_ips.erb
M modules/role/manifests/lvs/balancer.pp
6 files changed, 100 insertions(+), 173 deletions(-)
Approvals:
Giuseppe Lavagetto: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/lvs/manifests/balancer/runcommand.pp
b/modules/lvs/manifests/balancer/runcommand.pp
index fbf8a0d..94eeb8b 100644
--- a/modules/lvs/manifests/balancer/runcommand.pp
+++ b/modules/lvs/manifests/balancer/runcommand.pp
@@ -2,7 +2,6 @@
# Supporting the PyBal RunCommand monitor
class lvs::balancer::runcommand {
- Class[lvs::balancer] -> Class[lvs::balancer::runcommand]
file {
'/etc/pybal/runcommand':
diff --git a/modules/lvs/manifests/balancer.pp
b/modules/lvs/manifests/kernel_config.pp
similarity index 82%
rename from modules/lvs/manifests/balancer.pp
rename to modules/lvs/manifests/kernel_config.pp
index 5336768..146963d 100644
--- a/modules/lvs/manifests/balancer.pp
+++ b/modules/lvs/manifests/kernel_config.pp
@@ -1,22 +1,10 @@
-# lvs/balancer.pp
-
-# Class: lvs::balancer
-# Parameters:
-# - $service_ips: list of service IPs to bind to loopback
-# - $lvs_services: A configuration hash of LVS services
-# - $lvs_class_hosts: A configuration hash of PyBal class hosts
-class lvs::balancer(
- $lvs_services,
- $lvs_class_hosts,
- $service_ips=[],
- $conftool_prefix = '/conftool/v1',
- ) {
-
- include ::cpufrequtils # defaults to "performance", Ubuntu default is
"ondemand"
- include ::initramfs
+# Class: lvs::kernel_config
+#
+# Sets up kernel-level parameters for lvs
+#
+class lvs::kernel_config {
# ethtool is also a package needed but it is included from base
-
file { '/etc/modprobe.d/lvs.conf':
ensure => present,
owner => 'root',
@@ -24,9 +12,6 @@
content => template('lvs/lvs.conf.erb'),
notify => Exec['update-initramfs'],
}
-
- # Bind balancer IPs to the loopback interface
- class { '::lvs::realserver': realserver_ips => $service_ips }
sysctl::parameters { 'lvs':
values => {
@@ -80,4 +65,13 @@
mode => '0444',
content => "ip_vs\n",
}
+
+ # Bump min_free_kbytes a bit to ensure network buffers are available
quickly
+ vm::min_free_kbytes { 'lvs':
+ pct => 3,
+ min => 131072,
+ max => 524288,
+ }
+
+
}
diff --git a/modules/profile/manifests/lvs.pp b/modules/profile/manifests/lvs.pp
new file mode 100644
index 0000000..5461a11
--- /dev/null
+++ b/modules/profile/manifests/lvs.pp
@@ -0,0 +1,49 @@
+# === Class profile::lvs
+#
+# Sets up a linux load-balancer.
+#
+class profile::lvs {
+ require ::lvs::configuration
+
+ ## Kernel setup
+
+ # defaults to "performance", Ubuntu default is "ondemand"
+ class { '::cpufrequtils': }
+
+ # kernel-level parameters
+ class { '::lvs::kernel_config': }
+
+
+ ## LVS IPs setup
+ # Obtain all the IPs configured for this class of load-balancers,
+ # as a string. This is based on
+ # $::lvs::configuration::lvs_grain_class
+ # and
+ # $::lvs::configuration::lvs_services
+ $service_ips = template('profile/lvs/service_ips.erb')
+
+ # Bind balancer IPs to the loopback interface
+ class { '::lvs::realserver':
+ realserver_ips => $service_ips
+ }
+
+ # Monitoring sysctl
+ $rp_args = inline_template('<%= @interfaces.split(",").map{|x|
"net.ipv4.conf.#{x.gsub("_","/")}.rp_filter=0" if !x.start_with?("lo")
}.compact.join(",") %>')
+ nrpe::monitor_service { 'check_rp_filter_disabled':
+ description => 'Check rp_filter disabled',
+ nrpe_command => "/usr/lib/nagios/plugins/check_sysctl ${rp_args}",
+ }
+
+ salt::grain { 'lvs':
+ grain => 'lvs',
+ value => $lvs::configuration::lvs_grain,
+ replace => true,
+ }
+
+ salt::grain { 'lvs_class':
+ grain => 'lvs_class',
+ value => $lvs::configuration::lvs_grain_class,
+ replace => true,
+ }
+
+}
diff --git a/modules/profile/manifests/pybal.pp
b/modules/profile/manifests/pybal.pp
index 68e4a34..a1c932e 100644
--- a/modules/profile/manifests/pybal.pp
+++ b/modules/profile/manifests/pybal.pp
@@ -6,6 +6,7 @@
$conftool_prefix = hiera('conftool_prefix'),
$config_source = hiera('profile::pybal::config_source'),
$config_host = hiera('profile::pybal::config_host'),
+ $ganglia_clusters = hiera('ganglia_clusters'),
) {
requires_os('debian >= jessie')
@@ -51,4 +52,9 @@
class { '::pybal::monitoring': }
+ # Sites with MediaWiki appservers need runcommand
+ if $::site in keys($ganglia_clusters['appserver']['sites']) {
+ class { '::lvs::balancer::runcommand': }
+ }
+
}
diff --git a/modules/profile/templates/lvs/service_ips.erb
b/modules/profile/templates/lvs/service_ips.erb
new file mode 100644
index 0000000..ed8b2d2
--- /dev/null
+++ b/modules/profile/templates/lvs/service_ips.erb
@@ -0,0 +1,30 @@
+<%-
+# TODO: un-copy this from lvs/wikimedia-lvs-realserver.erb.
+
+def flatten_ips(ips)
+ result = []
+
+ if ips.is_a?(Hash)
+ ips.values.each do |v|
+ result += flatten_ips(v)
+ end
+ elsif ips.is_a?(Array)
+ ips.each do |v|
+ result += flatten_ips(v)
+ end
+ else
+ result << ips
+ end
+ return result
+end
+
+# TODO: maybe split primary/secondary and classes?
+lvs_class = scope.lookupvar("::lvs::configuration::lvs_grain_class")
+lvs_services = scope.lookupvar("::lvs::configuration::lvs_services")
+services = lvs_services.values.select do |srv|
+ # We select objects in all classes if class is "secondary"
+ (lvs_class == srv['class'] || lvs_class == 'secondary' ) &&
srv['sites'].include?(@site)
+end
+ips = flatten_ips(services.map{ |srv| srv['ip'][@site] })
+-%>
+<%= ips.uniq.sort.join(" ") -%>
diff --git a/modules/role/manifests/lvs/balancer.pp
b/modules/role/manifests/lvs/balancer.pp
index 51d3fd7..660f09b 100644
--- a/modules/role/manifests/lvs/balancer.pp
+++ b/modules/role/manifests/lvs/balancer.pp
@@ -1,161 +1,10 @@
class role::lvs::balancer {
system::role { 'lvs::balancer': description => 'LVS balancer' }
- $rp_args = inline_template('<%= @interfaces.split(",").map{|x|
"net.ipv4.conf.#{x.gsub("_","/")}.rp_filter=0" if !x.start_with?("lo")
}.compact.join(",") %>')
- nrpe::monitor_service { 'check_rp_filter_disabled':
- description => 'Check rp_filter disabled',
- nrpe_command => "/usr/lib/nagios/plugins/check_sysctl ${rp_args}",
- }
-
include ::lvs::configuration
-
- $sip = $lvs::configuration::service_ips
-
- # This is a temporary refactoring, we should do more to clean up here.
- # The whole point of $lvs_balancer_ips is to set up the current LVS node
- # puppet is executing on, yet we're subbing in $::site after selecting a
- # site-specific hostname, etc. There's also a great deal of redundancy
- # between information here and in configuration.
-
- $ips_eqiad_high_traffic1 = [ # IPs must be high-traffic1 subnet
- $sip['text'][$::site],
- ]
-
- $ips_eqiad_high_traffic2 = [ # IPs must be high-traffic2 subnet
- $sip['upload'][$::site],
- $sip['dns_rec'][$::site],
- $sip['misc_web'][$::site],
- $sip['git-ssh'][$::site],
- ]
-
- $ips_eqiad_low_traffic = [ # IPs must be low-traffic subnet
- $sip['apaches'][$::site],
- $sip['api'][$::site],
- $sip['rendering'][$::site],
- $sip['swift'][$::site],
- $sip['parsoid'][$::site],
- $sip['mathoid'][$::site],
- $sip['citoid'][$::site],
- $sip['cxserver'][$::site],
- $sip['search'][$::site],
- $sip['restbase'][$::site],
- $sip['zotero'][$::site],
- $sip['graphoid'][$::site],
- $sip['mobileapps'][$::site],
- $sip['kartotherian'][$::site],
- $sip['aqs'][$::site],
- $sip['eventbus'][$::site],
- $sip['apertium'][$::site],
- $sip['ores'][$::site],
- $sip['thumbor'][$::site],
- $sip['prometheus'][$::site],
- $sip['ocg'][$::site],
- $sip['wdqs'][$::site],
- $sip['kibana'][$::site],
- $sip['eventstreams'][$::site],
- $sip['pdfrender'][$::site],
- $sip['trendingedits'][$::site],
- $sip['kubemaster'][$::site],
- $sip['logstash'][$::site],
- ]
-
- $lvs_balancer_ips = $::hostname ? {
- # eqiad
- /^lvs100[147]$/ => $ips_eqiad_high_traffic1,
- /^lvs100[258]$/ => $ips_eqiad_high_traffic2,
- /^lvs100[369]$/ => $ips_eqiad_low_traffic,
- 'lvs1010' => array_concat(
- $ips_eqiad_high_traffic1,
- $ips_eqiad_high_traffic2,
- $ips_eqiad_low_traffic
- ),
-
- # codfw (should mirror eqiad above, eventually, and become merged with
it via regex
- /^(lvs200[14])$/ => [ # IPs must be high-traffic1 subnet
- $sip['text'][$::site],
- ],
- /^(lvs200[25])$/ => [ # IPs must be high-traffic2 subnet
- $sip['upload'][$::site],
- $sip['misc_web'][$::site],
- $sip['dns_rec'][$::site],
- ],
- /^(lvs200[36])$/ => [ # IPs must be low-traffic subnet
- $sip['apaches'][$::site],
- $sip['api'][$::site],
- $sip['rendering'][$::site],
- $sip['swift'][$::site],
- $sip['parsoid'][$::site],
- $sip['mathoid'][$::site],
- $sip['citoid'][$::site],
- $sip['cxserver'][$::site],
- $sip['search'][$::site],
- $sip['restbase'][$::site],
- $sip['zotero'][$::site],
- $sip['graphoid'][$::site],
- $sip['mobileapps'][$::site],
- $sip['apertium'][$::site],
- $sip['kartotherian'][$::site],
- $sip['eventbus'][$::site],
- $sip['ores'][$::site],
- $sip['thumbor'][$::site],
- $sip['prometheus'][$::site],
- $sip['wdqs'][$::site],
- $sip['eventstreams'][$::site],
- $sip['pdfrender'][$::site],
- $sip['trendingedits'][$::site],
- $sip['kubemaster'][$::site],
- ],
-
- # esams + ulsfo
- /^(lvs[34]00[13])$/ => [ # IPs must be high-traffic1 subnet
- $sip['text'][$::site],
- ],
- /^(lvs300[24])$/ => [ # IPs must be high-traffic2 subnet
- $sip['upload'][$::site],
- $sip['misc_web'][$::site],
- $sip['dns_rec'][$::site],
- ],
- /^(lvs400[24])$/ => [ # IPs must be high-traffic2 subnet
- $sip['upload'][$::site],
- $sip['misc_web'][$::site],
- ],
- }
-
include ::standard
- salt::grain { 'lvs':
- grain => 'lvs',
- value => $lvs::configuration::lvs_grain,
- replace => true,
- }
-
- salt::grain { 'lvs_class':
- grain => 'lvs_class',
- value => $lvs::configuration::lvs_grain_class,
- replace => true,
- }
-
- # TODO: refactor the whole set of classes
- class { '::lvs::balancer':
- service_ips => $lvs_balancer_ips,
- lvs_services => $lvs::configuration::lvs_services,
- lvs_class_hosts => $lvs::configuration::lvs_class_hosts,
- conftool_prefix => hiera('conftool_prefix'),
- }
-
include ::profile::pybal
+ include ::profile::lvs
- if $::site in ['eqiad', 'codfw'] {
- include ::lvs::balancer::runcommand
- }
-
- # production-only tweaks
- if $::realm == 'production' {
- # Bump min_free_kbytes a bit to ensure network buffers are available
quickly
- vm::min_free_kbytes { 'lvs':
- pct => 3,
- min => 131072,
- max => 524288,
- }
- }
}
--
To view, visit https://gerrit.wikimedia.org/r/357863
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I14581d7743abc09b61fed8a5f2cf7b95ea4ed440
Gerrit-PatchSet: 11
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Alexandros Kosiaris <akosia...@wikimedia.org>
Gerrit-Reviewer: BBlack <bbl...@wikimedia.org>
Gerrit-Reviewer: Ema <e...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
