Author: goneri-guest Date: 2008-04-17 22:23:00 +0000 (Thu, 17 Apr 2008) New Revision: 795
Modified: svnbuildstat/trunk/script/svnbuildstat_agent.pl Log: check for the free memory avalaible and kill the builds if needed, also some bug fixes and code rewrite. Modified: svnbuildstat/trunk/script/svnbuildstat_agent.pl =================================================================== --- svnbuildstat/trunk/script/svnbuildstat_agent.pl 2008-04-17 20:29:18 UTC (rev 794) +++ svnbuildstat/trunk/script/svnbuildstat_agent.pl 2008-04-17 22:23:00 UTC (rev 795) @@ -27,13 +27,12 @@ #my $maxjobs = $config->agent_maxjobs; my $maxjobs = 1; -my $vardir = "/home/goneri/svnbuildstattest"; +my $vardir = "/home/goneri/svnbuildstat"; my $workplace = $vardir."/workplace"; my $pbuilderplace = $vardir."/pbuilder"; #my $tarballsplace = $vardir."/tarballs"; my $buildarea = $vardir."/report"; my $tmpdir = $vardir."/tmp"; -my $ccachedir = $vardir."/ccache"; my $aptcachedir = $vardir."/aptcache"; my $debmirror = "http://ftp.fr.debian.org/debian/";; my $admin = "Gonéri Le Bouder <[EMAIL PROTECTED]>"; @@ -46,59 +45,92 @@ $ENV{LC_ALL} = 'C'; $ENV{LANG} = 'C'; -die "please create: `".$vardir unless -d $vardir; +sub purgeDir { + + die "please create: `".$vardir unless -d $vardir; # TODO check for running script in background # Purge -if (-d $workplace) {`rm -r $workplace`;} -if (-d $buildarea) {`rm -r $buildarea`;} -if (-d $tmpdir) {`rm -r $tmpdir`;} + foreach ($workplace, $buildarea, $tmpdir) { + if (-d $workplace) { + `rm -r $workplace`; + } + } + # Create the directorys -die if (!-d $workplace && !mkdir $workplace); -die if (!-d $buildarea && !mkdir $buildarea); -die if (!-d $pbuilderplace && !mkdir $pbuilderplace); -die if (!-d $tmpdir && !mkdir $tmpdir); -die if (!-d $ccachedir && !mkdir $ccachedir); -die if (!-d $ccachedir."/cache" && !mkdir $ccachedir."/cache"); -die if (!-d $aptcachedir && !mkdir $aptcachedir); -unlink foreach (<$tmpdir/report*.tar>); + foreach ($workplace, $buildarea, $pbuilderplace, $tmpdir, $aptcachedir) { + if (!-d && !mkdir $_) { + die "Failed to create $_\n"; + } + } +} -open (CCACHECFG, ">", "$ccachedir/ccache.cfg") or die; -print CCACHECFG "export CCACHE_DIR=\"$ccachedir/cache\"\n"; -print CCACHECFG "export PATH=\"/usr/lib/ccache:\${PATH}\"\n"; -print CCACHECFG "export CCACHE_NOCOMPRESS=\"1\"\n"; -print CCACHECFG "EXTRAPACKAGES=ccache\n"; -print CCACHECFG "BINDMOUNTS=\"\${CCACHE_DIR}\"\n"; -# --buildresult foo doesn't work? -#print CCACHECFG "BUILDRESULT=\"$tmpdir\"\n"; -close CCACHECFG; +sub checkSanity { + if (isMemFull()||isDiskFull($workplace)||isDiskFull($buildarea)||isDiskFull('/tmp')) { + print "checkSanity(): disk or memory full!!!\n"; + if (%job&&(isDiskFull($workplace)||isDiskFull($buildarea))) { + print "Emergency clean up\n"; + foreach (keys %job) { + kill 15, $_; + } + sleep (60*3); + foreach (keys %job) { + kill 9, $_; + waitpid($_, WNOHANG); + delete ($job{$_}); + } + + `rm -f $aptcachedir/*` unless $keeptmpdir; + purgeDir() unless $keeptmpdir; + } + } +} + sub postFile { my $file = shift; my $ua = new LWP::UserAgent; - + my $response = $ua->request(POST $server.'/controls/sendFile', Content_Type => 'form-data', Content => ['file' => [ $file, basename($file) ]]) or die "Upload failed\n"; - + print "report sent: ".$response->as_string()."\n"; } -sub isFull { +sub isDiskFull { my $path = shift; + my $free = 0; +# /dev/hda2 7210656 5721668 1122708 84% / foreach (`df -P $path`) { if (/^\S+\s+\d+\s+\d+\s+(\d+)\s+\S+/) { - return ($1 < 300000)?1:0; + $free = $1; } } + print "disk for $path: $free\n"; + + $free < 300000; +} - return; +sub isMemFull { + + if (!open MEMINFO, "</proc/meminfo") { + warn "Can't open meminfo"; + return; + } + + my $free = 0; + foreach (<MEMINFO>) { + $free += $2 if /^(MemFree|SwapFree):\s+(\d+)\s+kB/; + } + print "mem: ".$free."\n"; + # 50MB is not enough + $free < 50000; } - sub getBuildJob { my $p; my $report = {}; @@ -124,6 +156,8 @@ } sub upgradeChroot { + my $retry = shift; + $retry = 1 unless $retry; my $pdebuildparam = "--debootstrap debootstrap --mirror ".$debmirror." --buildplace $pbuilderplace --othermirror 'deb ".$debmirror." $distro main contrib non-free' --distribution $distro --basetgz $pbuildertgz --debootstrap debootstrap"; if (-s $pbuildertgz) { @@ -143,15 +177,19 @@ # I restore the previous chroot `cp $pbuildertgz.save $pbuildertgz`; `/usr/sbin/pbuilder update $pdebuildparam >/tmp/pbuilder-create.log 1>&2`; - } else { - # I save the created chroot - `cp $pbuildertgz $pbuildertgz.save`; } + } else { + # I save the created chroot + `cp $pbuildertgz $pbuildertgz.save`; } } if (! -s $pbuildertgz) { - print "No pbuilder tarball\n"; + my $timer = 10*$retry; + $timer = 180 if $timer > 180; + print "Failed to create a pbuilder tarball, will sleep for $timer minutes and retry ...\n"; + sleep ($timer*60); + upgradeChroot($retry+1); } } @@ -183,7 +221,7 @@ print BUILDLOG "(log file truncated) ... " if tell(BUILDLOGTMP); foreach (<BUILDLOGTMP>) { - # To avoid strange breakage I do some clean up in the log file + # To avoid strange breakage I do some clean up in the log file s/[[:cntrl:]]//g; print BUILDLOG $_."\n"; } @@ -250,8 +288,8 @@ $report->{'stamp_build-start'} = time; my $dsc_localpath = $threadworkplace.'/'.basename($report->{dsc}); - - `cd $threadworkplace/$tarballdir; pbuilder --build --buildresult $threadbuildarea --configfile $ccachedir/ccache.cfg --buildplace $pbuilderplace --aptcache $aptcachedir --distribution $distro --basetgz $pbuilderplace/$distro.tar.gz --timeout 10h $dsc_localpath >$threadbuildarea/build.log.tmp 2>&1`; + + `cd $threadworkplace/$tarballdir; pbuilder --build --buildresult $threadbuildarea --buildplace $pbuilderplace --aptcache $aptcachedir --distribution $distro --basetgz $pbuilderplace/$distro.tar.gz --timeout 10h $dsc_localpath >$threadbuildarea/build.log.tmp 2>&1`; $report->{'stamp_build-end'} = time; sendReport($report, $threadbuildarea); @@ -261,22 +299,15 @@ 1; } +################################################################## +################################################################## +################################################################## +###### Main +purgeDir(); + while (sleep 15) { # TODO replace by a more important values - if (isFull($workplace)||isFull($buildarea)) { - `rm -f $aptcachedir/*` unless $keeptmpdir; - print "disk is full\n"; - if (%job&&(isFull($workplace)||isFull($buildarea))) { - print "Emergency clean up\n"; - foreach (keys %job) { - kill 9, $_; - waitpid($_, WNOHANG); - delete ($job{$_}); - } - `rm -rf $workplace/* $buildarea/*` unless $keeptmpdir; - } - } - + checkSanity(); upgradeChroot(); # Check the running jobs (See #453710) @@ -290,7 +321,6 @@ } } - if (keys %job < $maxjobs) { my $newpid = fork; if ($newpid) { # Father @@ -306,5 +336,4 @@ exit; } } - } _______________________________________________ Collab-qa-commits mailing list [email protected] http://lists.alioth.debian.org/mailman/listinfo/collab-qa-commits
