BBlack has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/311239

Change subject: cron_splay() with first use in cache_upload
......................................................................

cron_splay() with first use in cache_upload

Change-Id: I995c8e55018bbd6544a55cae744658e972c72726
---
M modules/role/manifests/cache/upload.pp
M modules/varnish/templates/varnish-backend-restart.cron.erb
A modules/wmflib/lib/puppet/parser/functions/cron_splay.rb
3 files changed, 138 insertions(+), 4 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/39/311239/1

diff --git a/modules/role/manifests/cache/upload.pp 
b/modules/role/manifests/cache/upload.pp
index 908f7ad..7e828de 100644
--- a/modules/role/manifests/cache/upload.pp
+++ b/modules/role/manifests/cache/upload.pp
@@ -131,9 +131,12 @@
     }
 
     # XXX: temporary, we need this to mitigate T145661
-    $rt_parts = split(inline_template("<%= require 'digest/md5'; x = 
Random.new(Digest::MD5.hexdigest(@fqdn).to_i(16)).rand(1440); hh = x / 60; mm = 
x % 60; hh.to_s() + ':' + mm.to_s(); %>"), ':')
-    $be_restart_h = $rt_parts[0]
-    $be_restart_m = $rt_parts[1]
+    $hnodes = hiera('cache::upload::nodes')
+    $all_nodes = array_concat($hnodes['eqiad'], $hnodes['esams'], 
$hnodes['ulsfo'], $hnodes['codfw'])
+    $times = cron_splay($all_nodes, 'daily', 'upload-backend-restarts')
+    $be_restart_h = $times['hour']
+    $be_restart_m = $times['minute']
+    $be_restart_d = $times['weekday']
 
     file { '/etc/cron.d/varnish-backend-restart':
         mode    => '0444',
diff --git a/modules/varnish/templates/varnish-backend-restart.cron.erb 
b/modules/varnish/templates/varnish-backend-restart.cron.erb
index bc88893..2844280 100644
--- a/modules/varnish/templates/varnish-backend-restart.cron.erb
+++ b/modules/varnish/templates/varnish-backend-restart.cron.erb
@@ -1 +1 @@
-<%= @be_restart_m %> <%= @be_restart_h %> * * * root 
/usr/local/sbin/varnish-backend-restart > /dev/null
+<%= @be_restart_m %> <%= @be_restart_h %> * * <%= @be_restart_d %> root 
/usr/local/sbin/varnish-backend-restart > /dev/null
diff --git a/modules/wmflib/lib/puppet/parser/functions/cron_splay.rb 
b/modules/wmflib/lib/puppet/parser/functions/cron_splay.rb
new file mode 100644
index 0000000..7dba3e2
--- /dev/null
+++ b/modules/wmflib/lib/puppet/parser/functions/cron_splay.rb
@@ -0,0 +1,131 @@
+#
+# cron_splay.rb
+#
+
+require 'digest/md5'
+
+module Puppet::Parser::Functions
+  newfunction(:cron_splay, :type => :rvalue, :doc => <<-EOS
+Given an array of fqdn which a cron is applicable to, and a period arg which is
+one of 'hourly', 'daily', or 'weekly', this sorts the fqdn set with
+per-datacenter interleaving for DC-numbered hosts, splays them to fixed even
+intervals within the total period, and then outputs a set of crontab time
+fields for the fqdn currently being compiled-for.
+
+The idea here is to ensure each host in the set executes the cron once per time
+period, and also ensure the time between hosts is consistent (no edge cases
+much closer than the average) by splaying them as evenly as possible with
+rounding errors.  For the case of hosts with NNNN numbers indicating the
+datacenter in the first digit, we also maximize the period between any two
+hosts in a given datacenter by interleaving sorted per-DC lists of hosts before
+splaying.
+
+The third and final argument is a static seed which modulates the splayed
+values in two different ways to minimize the effects of multiple cron_splay()
+with the same hostlist and period.  It is used to select a
+determinstically-random "offset" for the splayed time values (so that the first
+host doesn't always start at 00:00), and is also used to permute the order of
+the hosts within each DC uniquely.
+
+*Examples:*
+
+    $times = fqdn_splay($hosts, 'weekly', 'foo-static-seed')
+    crontab { 'foo':
+        minute   => $times['minute'],
+        hour     => $times['hour'],
+        weekday  => $times['weekday'],
+    }
+
+    EOS
+  ) do |arguments|
+
+    raise(Puppet::ParseError, "cron_splay(): Wrong number of arguments " +
+      "given (#{arguments.size} for 3)") if arguments.size != 3
+
+    hosts = arguments[0]
+    period = arguments[1]
+    seed = arguments[2]
+
+    unless hosts.is_a?(Array)
+      raise(Puppet::ParseError, 'cron_splay(): Argument 1 must be an array')
+    end
+
+    unless period.is_a?(String)
+      raise(Puppet::ParseError, 'cron_splay(): Argument 2 must be an string')
+    end
+
+    unless seed.is_a?(String)
+      raise(Puppet::ParseError, 'cron_splay(): Argument 3 must be an string')
+    end
+
+    case period
+      when 'hourly'
+         mins = 60
+      when 'daily'
+         mins = 1440
+      when 'weekly'
+         mins = 10080
+      else
+        raise(Puppet::ParseError, 'cron_splay(): invalid period')
+    end
+
+    # Avoid this edge case for now.  At sufficiently large host counts and
+    # small period, randomization is probably better anyways.
+    if hosts.length > mins
+      raise(Puppet::ParseError, 'cron_splay(): too many hosts for period')
+    end
+
+    # split hosts into N lists based the first digit of /NNNN/, defaulting to 
zero
+    sublists = [ [], [], [], [], [], [], [], [], [], [] ]
+    for h in hosts
+      match = /([1-9])[0-9]{3}/.match(h)
+      if match
+        sublists[match[1]].push(h)
+      else
+        sublists[0].push(h)
+      end
+    end
+
+    # sort each sublist into a determinstic order based on seed
+    for s in sublists
+      s.sort_by! { |x| Digest::MD5.hexdigest(seed + x) }
+    end
+
+    # interleave sublists into "ordered"
+    longest = sublists.max_by(&:length)
+    sublists -= [longest]
+    ordered = longest.zip(*sublists).flatten.compact
+
+    # find the index of this host in ordered
+    this_idx = ordered.index(lookupvar('::fqdn'))
+    if this_idx == nil
+      raise(Puppet::ParseError, 'cron_splay(): this host not in set')
+    end
+
+    # find the truncated-integer splayed value of this host
+    tval = this_idx * mins / ordered.length
+
+    # use the seed (again) to add a time offset to the splayed values,
+    # the time offset never being larger than the splayed interval
+    tval += Digest::MD5.hexdigest(seed).to_i(16) % (mins / ordered.length)
+
+    # generate the output
+    output['minute'] = tval % 60
+
+    if period == 'hourly'
+      outout['hour'] = '*'
+    else
+      output['hour'] = ( tval / 60 ) % 24
+    end
+
+    if period == 'weekly'
+      output['weekday'] = tval / 1440
+    else
+      output['weekday'] = '*'
+    end
+
+    return output
+  end
+end
+
+# vim: set ts=2 sw=2 et :

-- 
To view, visit https://gerrit.wikimedia.org/r/311239
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I995c8e55018bbd6544a55cae744658e972c72726
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack <bbl...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to