Rush has submitted this change and it was merged. Change subject: labstore: stage tc script and enable on boot ......................................................................
labstore: stage tc script and enable on boot We have been piloting this configuration for a few weeks to limit the variable load exposure of labstore10* in tools. Labstore1001 especially has difficulty sustaining the extreme bursty load that any operation can trigger. These shaping values are based on data collected from the nfsiostat diamond collector deployed a few weeks ago. This will all most likely require tweaking over time. Change-Id: I19814c3da53f78666a8877937353f3987a904501 --- A modules/labstore/files/tc-setup.sh A modules/labstore/manifests/traffic_shaping.pp M modules/role/manifests/labs/instance.pp 3 files changed, 139 insertions(+), 0 deletions(-) Approvals: Andrew Bogott: Looks good to me, but someone else must approve Rush: Looks good to me, approved jenkins-bot: Verified diff --git a/modules/labstore/files/tc-setup.sh b/modules/labstore/files/tc-setup.sh new file mode 100644 index 0000000..f7f5082 --- /dev/null +++ b/modules/labstore/files/tc-setup.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +# this script applies traffic shaping using tc. +# it is intended to target NFS traffic, but since +# the NFS hosts do not offer other services it is +# applied based on IP. + +# intended as idempotent + +# tc -s qdisc show +# http://lartc.org/manpages/tc.txt + +modules='act_mirr ifb' +nfs_write='7000kbps' +nfs_read='9500kbps' +eth0_egress='30000kbps' + +function ensure_mod { + value=$(/sbin/lsmod | /bin/grep $1) + retcode=$? + if [ $retcode != 0 ] + then + echo "$1 is not loaded" + exit 1 + fi +} + +for m in $modules; do + ensure_mod $m +done + +TC=$(which tc) + +# clear out old config +$TC qdisc del dev eth0 root + +$TC qdisc add dev eth0 root handle 1: htb default 100 + +$TC class add dev eth0 parent 1: classid 1:1 htb rate $nfs_write + +$TC class add dev eth0 parent 1: classid 1:2 htb rate $nfs_write + +$TC class add dev eth0 parent 1: classid 1:3 htb rate $nfs_write + +$TC class add dev eth0 parent 1: classid 1:100 htb rate $eth0_egress + +$TC filter add dev eth0 parent 1: protocol ip prio 0 u32 \ + match ip dst 10.64.37.6 flowid 1:1 + +$TC filter add dev eth0 parent 1: protocol ip prio 0 u32 \ + match ip dst 10.64.37.7 flowid 1:2 + +$TC filter add dev eth0 parent 1: protocol ip prio 0 u32 \ + match ip dst 10.64.37.10 flowid 1:3 + +#------------------------------------- + +# clear out config +$TC qdisc del dev eth0 handle ffff: ingress +$TC qdisc del dev ifb0 root + +# Create ingress on external interface +$TC qdisc add dev eth0 handle ffff: ingress + +# this link has to come up for ingress shaping +/sbin/ip link set dev ifb0 up +retcode=$? +if [ $retcode != 0 ] +then + echo "ifb0 is not coming up" + $TC qdisc del dev eth0 handle ffff: ingress + $TC qdisc del dev ifb0 root + exit 1 +fi + +# pass engress traffic through ifb0 +$TC filter add dev eth0 parent ffff: protocol all u32 \ + match u32 0 0 action mirred egress redirect dev ifb0 + +$TC qdisc add dev ifb0 root handle 1: htb + +$TC class add dev ifb0 parent 1: classid 1:1 htb rate $nfs_read + +$TC class add dev ifb0 parent 1: classid 1:2 htb rate $nfs_read + +$TC class add dev ifb0 parent 1: classid 1:3 htb rate $nfs_read + +$TC filter add dev ifb0 parent 1: protocol ip prio 0 u32 \ + match ip src 10.64.37.6 flowid 1:1 + +$TC filter add dev ifb0 parent 1: protocol ip prio 0 u32 \ + match ip src 10.64.37.7 flowid 1:2 + +$TC filter add dev ifb0 parent 1: protocol ip prio 0 u32 \ + match ip src 10.64.37.10 flowid 1:3 diff --git a/modules/labstore/manifests/traffic_shaping.pp b/modules/labstore/manifests/traffic_shaping.pp new file mode 100644 index 0000000..df9f55c --- /dev/null +++ b/modules/labstore/manifests/traffic_shaping.pp @@ -0,0 +1,37 @@ +class labstore::traffic_shaping { + + file { '/usr/local/sbin/tc-setup': + ensure => present, + mode => '0554', + owner => 'root', + group => 'root', + source => 'puppet:///modules/labstore/tc-setup.sh', + } + + # run when interfaces come up. + file { '/etc/network/if-up.d/tc': + ensure => 'link', + target => '/usr/local/sbin/tc-setup', + } + + # under systemd either /etc/modules or /etc/load-modules.d works + # since labs still has precise instances this is applied + # using the non-.d model since it is still effective and consistent + file_line { 'enable_ifb': + ensure => present, + line => 'ifb', + path => '/etc/modules', + } + + file_line { 'enable_act_mirred': + ensure => present, + line => 'act_mirred', + path => '/etc/modules', + } + + # ifb by default creates 2 interfaces + file { '/etc/modprobe.d/ifb.conf': + ensure => present, + content => 'options ifb numifbs=1', + } +} diff --git a/modules/role/manifests/labs/instance.pp b/modules/role/manifests/labs/instance.pp index 0ce29fd..41c2d09 100644 --- a/modules/role/manifests/labs/instance.pp +++ b/modules/role/manifests/labs/instance.pp @@ -62,6 +62,7 @@ } if mount_nfs_volume($::labsproject, 'home') and $mount_nfs { + include labstore::traffic_shaping # Note that this is the same export as for /data/project exec { 'block-for-home-export': command => "/usr/local/sbin/block-for-export ${nfs_server} project/${::labsproject} 180", @@ -80,6 +81,7 @@ } if mount_nfs_volume($::labsproject, 'project') or mount_nfs_volume($::labsproject, 'scratch') { + include labstore::traffic_shaping # Directory for data mounts file { '/data': ensure => directory, @@ -90,6 +92,7 @@ } if mount_nfs_volume($::labsproject, 'project') and $mount_nfs { + include labstore::traffic_shaping exec { 'block-for-project-export': command => "/usr/local/sbin/block-for-export ${nfs_server} project/${::labsproject} 180", require => [File['/etc/modprobe.d/nfs-no-idmap'], File['/usr/local/sbin/block-for-export']], @@ -112,6 +115,7 @@ } if mount_nfs_volume($::labsproject, 'scratch') and $mount_nfs { + include labstore::traffic_shaping # We don't need to block for this one because it's always exported for everyone. file { '/data/scratch': ensure => directory, @@ -129,6 +133,7 @@ } if mount_nfs_volume($::labsproject, 'dumps') or mount_nfs_volume($::labsproject, 'statistics') { + include labstore::traffic_shaping # Directory for public (readonly) mounts file { '/public': ensure => directory, @@ -139,6 +144,7 @@ } if mount_nfs_volume($::labsproject, 'statistics') and $mount_nfs { + include labstore::traffic_shaping file { '/public/statistics': ensure => directory, require => File['/public'], @@ -155,6 +161,7 @@ } if mount_nfs_volume($::labsproject, 'dumps') and $mount_nfs { + include labstore::traffic_shaping file { '/public/dumps': ensure => directory, require => File['/public'], -- To view, visit https://gerrit.wikimedia.org/r/272891 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I19814c3da53f78666a8877937353f3987a904501 Gerrit-PatchSet: 4 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Rush <[email protected]> Gerrit-Reviewer: Andrew Bogott <[email protected]> Gerrit-Reviewer: Rush <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
