Tim Landscheidt has uploaded a new change for review. https://gerrit.wikimedia.org/r/234051
Change subject: Tools: Decommission bigbrother ...................................................................... Tools: Decommission bigbrother Change-Id: I9483d54d59e0e3210093bfb96ead463b61d4eea6 --- D modules/toollabs/files/bigbrother D modules/toollabs/files/bigbrother.conf M modules/toollabs/manifests/submit.pp 3 files changed, 11 insertions(+), 252 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/51/234051/1 diff --git a/modules/toollabs/files/bigbrother b/modules/toollabs/files/bigbrother deleted file mode 100755 index 7b5ab58..0000000 --- a/modules/toollabs/files/bigbrother +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/perl -w -# -# Copyright © 2014 Marc-André Pelletier <[email protected]> -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -use XML::Simple qw/:strict/; -use Date::Parse; -use POSIX qw/strftime/; -use File::stat; - -use Data::Dumper; - -my %users; - -sub ulog($$$) { - my($username, $type, $msg) = @_; - my $when = strftime "%Y-%m-%d %H:%M:%S", gmtime; - - return unless defined $users{$username}; - - if(open LOG, ">>$users{$username}->{logfile}") { - print LOG "$when $type: $msg\n"; - close LOG; - } - if(open EMAIL, "|/usr/bin/sudo -iu $username /usr/sbin/exim -odf -t") { - print EMAIL "Subject: [bigbrother] $type: $msg\n"; - print EMAIL "To: $username maintainers <$username>\n"; - print EMAIL "From: Bigbrother <$username>\n"; - print EMAIL "\n"; - print EMAIL "$when $type: $msg\n"; - close EMAIL; - } -} - -sub readconfig($) { - my($username) = @_; - my $now = time; - - unless(defined $users{$username}) { - my @u = getpwnam $username; - return unless $u[0] eq $username; - return unless defined $u[7]; - - $users{$username} = { - rcfile => "$u[7]/.bigbrotherrc", - logfile => "$u[7]/bigbrother.log", - timestamp => 0, - refresh => $now, - }; - } - - return if $now < $users{$username}->{refresh}; - $users{$username}->{refresh} = $now + int(rand(60)); - - my $rcfile = $users{$username}->{rcfile}; - my $sb = stat($rcfile); - return unless defined $sb; - return if $sb->mtime <= $users{$username}->{timestamp}; - $users{$username}->{timestamp} = $sb->mtime; - - if(open RC, "<$rcfile") { - my %nj; - - my $line = 0; - while(<RC>) { - $line++; - my $expect; - my $start; - if (m/^\s*(?:#.*)?$/) { # Ignore empty lines and comments. - next; - } elsif(m/^webservice/) { - next; # Ignore webservice lines, they are taken care of by service manifests - } elsif(m/^jstart\s+-N\s+(\S+)\s+(.*)$/) { - $expect = $1; - $start = "/usr/bin/jstart -N '$expect' $2"; - } else { - ulog $username, 'error', "$rcfile:$line: command not supported"; - next; - } - if(defined $nj{$expect}) { - ulog $username, 'warn', "$rcfile:$line: duplicate job name '$expect' ignored"; - next; - } - $nj{$expect} = { - start => $start, - jname => $expect, - }; - } - close RC; - - if(defined $users{$username}->{jobs}) { - foreach my $jn (keys %{$users{$username}->{jobs}}) { - undef $users{$username}->{jobs}->{$jn}->{start}; - } - foreach my $jn (keys %nj) { - if(defined $users{$username}->{jobs}->{$jn}) { - $users{$username}->{jobs}->{$jn}->{start} = $nj{$jn}->{start}; - } else { - $users{$username}->{jobs}->{$jn} = $nj{$jn}; - } - } - foreach my $jn (keys %{$users{$username}->{jobs}}) { - undef $users{$username}->{jobs}->{$jn} unless defined $users{$username}->{jobs}->{$jn}->{start}; - } - } else { - $users{$username}->{jobs} = \%nj if scalar keys %nj; - } - } -} - -sub start($$) { - my($username, $job) = @_; - my $now = time; - - $job->{starts} = [ ] unless defined $job->{starts}; - while($#{$job->{starts}}>=0 and $job->{starts}->[0]<($now - 60*60*24)) { - shift @{$job->{starts}}; - } - - # Jobs are allowed 10 attempts to start in a 24h window - if($#{$job->{starts}} > 10) { - ulog $username, 'warn', "Too many attempts to restart job '$job->{jname}'; throttling"; - $job->{state} = 'throttled'; - $job->{since} = $now; - $job->{timeout} = $job->{starts}->[0] + 60*60*24; - return; - } - - push @{$job->{starts}}, $now; - - $job->{state} = 'starting'; - $job->{since} = $now; - $job->{timeout} = $now + 90 + int(rand(30)); - ulog $username, 'info', "Restarting job '$job->{jname}'"; - if(open PIPE, "|/usr/bin/sudo -iu '$username' >>$users{$username}->{logfile} 2>&1") { - print PIPE $job->{start}, "\n"; - close PIPE; - } -} - -my $scoreboard = "/data/project/.system/bigbrother.scoreboard"; - -for(;;) { - - my $file = IO::File->new(); - if($file->open("/usr/bin/qstat -u '*' -xml|")) { - my $xml = XMLin($file, - KeyAttr => { }, - ForceArray => [ 'job_list' ], - ); - - foreach my $username (keys %users) { - next unless defined $users{$username}->{jobs}; - foreach my $je (values %{$users{$username}->{jobs}}) { - next if defined $je->{timeout}; - $je->{state} = 'dead'; - } - } - - foreach my $j (@{$xml->{queue_info}->{job_list}}, @{$xml->{job_info}->{job_list}}) { - my $jnum = $j->{JB_job_number}; - my $jname = $j->{JB_name}; - my $state = $j->{state}->[0]; - my $flags = $j->{state}->[1]; - my $owner = $j->{JB_owner}; - my $since = str2time($j->{JAT_start_time} // $j->{JB_submission_time}); - - readconfig $owner; - next unless defined $users{$owner}->{jobs}; - - my $je = $users{$owner}->{jobs}->{$jname}; - if(defined $je) { - $je->{jnum} = $jnum; - $je->{jname} = $jname; - $je->{state} = $state; - $je->{flags} = $flags; - $je->{since} = $since; - undef $je->{timeout}; - } - } - } - undef $file; - - open SB, ">$scoreboard~"; - print SB scalar(time), "\n"; - foreach my $username (keys %users) { - next unless defined $users{$username}->{jobs}; - foreach my $job (values %{$users{$username}->{jobs}}) { - next unless defined $job->{state}; - if($job->{state} eq 'dead') { - start $username, $job; - } elsif($job->{state} eq 'starting' or $job->{state} eq 'throttled') { - if(time >= $job->{timeout}) { - ulog $username, 'warn', "job '$job->{jname}' failed to start"; - start $username, $job; - } - } - print SB "$username:", - $job->{jname}, ":", - $job->{state} // 'unknown', ":", - $job->{since} // 0, ":", - $job->{flags} // '', ":", - $job->{timeout} // 0, "\n"; - } - } - close SB; - rename "$scoreboard~", "$scoreboard"; - - sleep 10; -} - diff --git a/modules/toollabs/files/bigbrother.conf b/modules/toollabs/files/bigbrother.conf deleted file mode 100644 index dea49a1..0000000 --- a/modules/toollabs/files/bigbrother.conf +++ /dev/null @@ -1,12 +0,0 @@ -# bigbrother - watches jobs and restarts them - -description "Bigbrother: watches jobs and restarts them" - -start on runlevel [2345] -stop on runlevel [!2345] - -respawn - -console log - -exec /usr/local/sbin/bigbrother diff --git a/modules/toollabs/manifests/submit.pp b/modules/toollabs/manifests/submit.pp index ffc7a6d..8e33cae 100644 --- a/modules/toollabs/manifests/submit.pp +++ b/modules/toollabs/manifests/submit.pp @@ -2,7 +2,7 @@ # # This role sets up an submit host instance in the Tool Labs model. # (A host that can only be used to submit jobs; presently used by -# tools-submit which runs bigbrother and the gridwide cron. +# tools-submit which runs the gridwide cron. # # Parameters: # @@ -44,25 +44,19 @@ ensure => latest, } - file { '/usr/local/sbin/bigbrother': - ensure => file, - owner => 'root', - group => 'root', - mode => '0555', - source => 'puppet:///modules/toollabs/bigbrother', + # TODO: Remove after decommissioning. + file { ['/data/project/.system/bigbrother.scoreboard', + '/data/project/.system/bigbrother.scoreboard~', + '/etc/init/bigbrother.conf', + '/usr/local/sbin/bigbrother']: + ensure => absent, + require => Service['bigbrother'], } - file { '/etc/init/bigbrother.conf': - ensure => file, - owner => 'root', - group => 'root', - mode => '0444', - source => 'puppet:///modules/toollabs/bigbrother.conf', - } - + # TODO: Remove after decommissioning. service { 'bigbrother': - ensure => running, - subscribe => File['/usr/local/sbin/bigbrother', '/etc/init/bigbrother.conf'], + ensure => stopped, + enable => false, } file { '/usr/local/bin/webservice2': -- To view, visit https://gerrit.wikimedia.org/r/234051 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I9483d54d59e0e3210093bfb96ead463b61d4eea6 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Tim Landscheidt <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
