Dzahn has uploaded a new change for review. https://gerrit.wikimedia.org/r/53612
Change subject: add Jeff's apache-fast-test script to repo ...................................................................... add Jeff's apache-fast-test script to repo Change-Id: I68095bba0654d55a6e586ad2dd329968ea100385 --- A files/misc/scripts/apache-fast-test 1 file changed, 199 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/12/53612/1 diff --git a/files/misc/scripts/apache-fast-test b/files/misc/scripts/apache-fast-test new file mode 100755 index 0000000..743baf0 --- /dev/null +++ b/files/misc/scripts/apache-fast-test @@ -0,0 +1,199 @@ +#!/usr/bin/perl +# +# Copyright (c) 2012 Jeff Green <[email protected]> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +use strict; +no warnings 'threads'; +use threads; +use threads::shared; +use LWP::UserAgent; +use Net::DNS::Resolver; +use Time::HiRes; + +my $threads = 50; # how many concurrent threads? +my $timeout_limit = 3; # how many timeouts to tolerate before dropping a server +my $timeout = 10; # seconds before LWP gives up on a web request +my $pad_length = 25; # server hostname column width in report +my $default_staging_server = 'srv193.pmtpa.wmnet'; + +# get a list of servers to test +my $servers; +if (@ARGV[1]) { + # explicit list of servers + for (@ARGV[1..$#ARGV]) { + chomp; + if (/^pybal$/) { + # list of production servers from pybal config + $servers = get_server_list_from_pybal_config(qw( + /home/w/conf/pybal/pmtpa/apaches + )); + } elsif (/^([\w\.]+)$/) { + $servers->{$1} = 1; + } + } +} else { + $servers->{$default_staging_server} = 1; +} + +# thread-shared variables used for test/result +my @queue : shared; # job_ids grouped in per-thread batches +my %result :shared; # threads return stuff here +my %timeouts :shared; # server timeout counter + +# read in the list of urls +my $urls; +open URL, $ARGV[0]; +while (<URL>) { + chomp; + if (/^\s*(http\S+)/) { + $urls->{$1}++; + } +} +close URL; + +unless (keys %{$urls}) { + print "[31mno urls found in file $ARGV[0][0m\n"; +} + +# do the server DNS lookups in advance, threading DNS lookups is fraught with peril +my $resolver = Net::DNS::Resolver->new; +for my $host (sort keys %{$servers}) { + my $answer = $resolver->search($host); + if ($answer) { + for my $r ($answer->answer) { + if ($r->type eq 'A') { + my $batch; + $servers->{$host} = $r->address; # grab the first A record + for my $url (sort keys %{$urls}) { + $batch .= "$servers->{$host}\t$url\n"; + } + push @queue, $batch if defined $batch; + last; + } + } + } + unless ($servers->{$host} > 1) { + print "[31mno IP found for $host[0m\n"; + delete $servers->{$host}; + } +} + +# informative output +if (scalar @queue) { + print "testing " . (keys %{$urls}) . ' urls on ' . (keys %{$servers}) . ' servers, totalling ' . + ((keys %{$urls}) * (keys %{$servers})) . " requests\n"; +} else { + print "\n usage: $0 url_file [server spec]\n\n" . + " url_file format is one URL per line, http/https schemes supported\n\n" . + " server spec:\n" . + " (none) use default $default_staging_server\n" . + " pybal fetch webserver list from local pybal conf\n" . + " host1 host2 host3\n\n"; + exit; +} + +# spawn worker threads to do api web requests, then reap them +print "spawning threads"; +for (1..$threads) { + last unless scalar @queue; + threads->new(sub { process_queue_task() }); +} +while (threads->list) { + sleep 0.1; + for my $thr (threads->list(threads::joinable)) { + $thr->join; + } +} +print "\n"; + +# can't nest hashes with threads::shared, so we retrieve from shared %result hash +print "\n"; +for my $url (sort keys %{$urls}) { + my ($highlight, $previous_result, $report); + for my $host (sort keys %{$servers}) { + my $ip = $servers->{$host}; + next unless defined $result{"$ip\t$url"}; + $report .= " " . sprintf("%-${pad_length}s",$host) . $result{"$ip\t$url"} . "\n"; + $highlight++ if defined($previous_result) and ($previous_result ne $result{"$ip\t$url"}); + $previous_result = $result{"$ip\t$url"}; + } + if (defined $highlight) { + print "[31m$url\n$report[0m"; + } else { + print "$url\n * $previous_result\n"; # short form output + } +} + +exit; + + + + + + +# SUBROUTINES +sub process_queue_task { + my $name = 'thread' . threads->self->tid(); + print '.'; + while (defined $queue[0]) { + for my $job (split("\n", shift @queue)) { + my ($ip,$url) = split("\t", $job); + if (($timeouts{$ip} < $timeout_limit) and ($url =~ /^(https?):\/\/([^\/]+)(.*)$/)) { + my ($protocol,$host,$path) = ($1,$2,$3); + $path = (defined $path) ? $path : ''; + my $ua = LWP::UserAgent->new(timeout => $timeout); + my $request = HTTP::Request->new(GET => "http://$ip$path";); + $request->header( + 'Host' => $host, + 'User_Agent' => $0, + 'X-Forwarded-Proto' => $protocol, # proxies add this + ); + my $r = $ua->simple_request($request); + if ($r->is_success) { + $result{"$ip\t$url"} = $r->status_line . ' ' . length($r->content); + } elsif (($r->is_error) and ($r->status_line =~ /^500 can't connect/i)) { # simple timeout, drop the data + $timeouts{$ip}++; + print "$name dropped $ip, too many timeouts\n" if $timeouts{$ip} >= $timeout_limit; + } elsif ($r->is_error) { # report other errors + $result{"$ip\t$url"} = $r->status_line; + } else { + $result{"$ip\t$url"} = $r->status_line . ' ' . $r->header('Location'); + } + } + } + } +} + + +sub get_server_list_from_pybal_config { + my $servers; + for my $pyfile (@_) { + open PYFILE, $pyfile; + while (<PYFILE>) { + next if /^\s*#/; + if ( (/'enabled':\s+True/i) and (/'host':\s+'([\.\w]+)'/) ) { + $servers->{$1} = 1; + } + } + close PYFILE; + } + return $servers; +} -- To view, visit https://gerrit.wikimedia.org/r/53612 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I68095bba0654d55a6e586ad2dd329968ea100385 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Dzahn <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
