Diederik has submitted this change and it was merged.
Change subject: Wikistats now expects ip field with country code
......................................................................
Wikistats now expects ip field with country code
* Updated code in SquidCountArchiveProcessLogRecord.pm to
expect country code in $fields[4] like "88.88.88.88|US"
* Updated tests to cope with this change
* Updated another time t/CommonConfig.pm to use ENV{WORKSPACE} needed
by jenkins
Change-Id: I9856f648a6e8013d264ef9c05d5a571d0703040f
---
M squids/perl/SquidCountArchiveProcessLogRecord.pm
M squids/perl/SquidCountArchiveReadInput.pm
M squids/t/06-regression-mismatch-world-north-south-unknown.t
M squids/t/09-regression-totals-fixes-squidreportclients.t
M squids/t/CommonConfig.pm
M squids/t/Generate/Squid.pm
6 files changed, 20 insertions(+), 62 deletions(-)
Approvals:
Diederik: Verified; Looks good to me, approved
jenkins-bot: Verified
diff --git a/squids/perl/SquidCountArchiveProcessLogRecord.pm
b/squids/perl/SquidCountArchiveProcessLogRecord.pm
index 7d32ef1..d537512 100755
--- a/squids/perl/SquidCountArchiveProcessLogRecord.pm
+++ b/squids/perl/SquidCountArchiveProcessLogRecord.pm
@@ -97,7 +97,8 @@
$redirected_to_mobile += $count_event ;
return ;
}
- }
+ };
+
$url =~ s/^http\w?\:\/\///o ;
$url =~ s/\%3A/:/gio ;
@@ -142,58 +143,12 @@
else
{ $mimecat = "other" ; }
+
+
if ($job_runs_on_production_server)
{
$country = "" ;
-
- #################################################
- # udp-filters has logic for gelocating and it's using
- # the x-forwarded-for field to geolocate first if that's
- # present but only the first ip inside that field.
- #
- # At the present moment there is not enough time to
- # check all the ips in XFF field but we can come back
- # at this later -- Stefan Petrea
- #
- # The reason for this is that now 2012-Dec-04 we have a high
- # priority for getting editor reports for November out
- # there.
- #################################################
-
- #@xffparts = split('%20',$xff) ;
- #foreach $ip (@xffparts)
- #{
- #next if $country ne "" ;
- #if ($ip =~ /^\d+\.\d+\.\d+\.\d+$/)
- #{
- #$country = $savedipcountry { $ip } ;
- #if ($country eq "")
- #{
- #$country = `echo $ip | /usr/local/bin/geoiplogtag 1` ;
- #$country =~ s/.*\s([\w-\(])/$1/ ;
- #$country =~ s/\s//g ;
- #$savedipcountry { $ip } = $country ;
- #}
- #$foundip = $ip ;
- #if ($country =~ /(^(--|-P|A1|A2|AB|BL|G2|GF|KO|MF|O1|TE|TK)$|null)/
) # Non-countries
- #{ $country = "" ; }
- #}
- #}
- if ($country eq "")
- {
- $country = $fields [14] ;
- $foundip = $client_ip ;
- }
- if (($country eq "") || ($country =~ /null/))
- {
- $country = "--" ;
- if ($foundip =~ /:/)
- {
- $country = "-P" ;
- }
- }
- if (&IsInternal($foundip))
- { $country = "-X" ; }
+ ($country) = $client_ip =~ /\|(..)$/;
}
else
{
diff --git a/squids/perl/SquidCountArchiveReadInput.pm
b/squids/perl/SquidCountArchiveReadInput.pm
index 6e8ad7d..d578444 100755
--- a/squids/perl/SquidCountArchiveReadInput.pm
+++ b/squids/perl/SquidCountArchiveReadInput.pm
@@ -182,12 +182,12 @@
else
{ open IN,"<$file_in" ; } # http://perldoc.perl.org/functions/open.html
- $fields_expected = 14 ;
+ $fields_expected = 13 ;
}
else
{
open IN, '<', $file_in ;
- $fields_expected = 14 ; # add fake country code
+ $fields_expected = 13 ; # add fake country code
# $fields_expected = 13 ;
}
@@ -220,7 +220,7 @@
# print "mime " . $fields[10] . "\n" ;
#next if $fields [9] eq '-' ;
#next if $fields [9] =~ /NONE/ ;
- if ($#fields > 14)
+ if ($#fields >= 13)
{
if (! $scan_ip_frequencies)
{
@@ -228,11 +228,7 @@
# print "fields " . $#fields . "\n$line\n" ;
}
- $country_code = $fields [$#fields] ;
- $fields [$#fields] = '' ;
- $line = join (' ', @fields) ;
@fields = split (' ', $line, 14) ;
- $fields [14] = $country_code ;
$fields [13] =~ s/ /%20/g ;
if (! $scan_ip_frequencies)
diff --git a/squids/t/06-regression-mismatch-world-north-south-unknown.t
b/squids/t/06-regression-mismatch-world-north-south-unknown.t
index 8f6b015..d8a3933 100644
--- a/squids/t/06-regression-mismatch-world-north-south-unknown.t
+++ b/squids/t/06-regression-mismatch-world-north-south-unknown.t
@@ -134,6 +134,10 @@
unknown = $unknown
";
+ok($world_total > 2, "world_total > 2");
+ok($global_south > 2, "globla_south > 2");
+ok($global_north > 2, "globla_north > 2");
+
warn "world_total - global_north - global_south - ipv6 - unknown =
".($world_total - $global_north - $global_south - $ipv6 - $unknown);
my $country_code_invariant_1 = $world_total - $global_north - $global_south -
$ipv6 - $unknown;
diff --git a/squids/t/09-regression-totals-fixes-squidreportclients.t
b/squids/t/09-regression-totals-fixes-squidreportclients.t
index 5d5fa95..7b533af 100644
--- a/squids/t/09-regression-totals-fixes-squidreportclients.t
+++ b/squids/t/09-regression-totals-fixes-squidreportclients.t
@@ -198,7 +198,7 @@
};
my $wikistats_run_cmd_output = `$wikistats_run_cmd`;
-warn $wikistats_run_cmd_output;
+#warn $wikistats_run_cmd_output;
use HTML::TreeBuilder::XPath;
diff --git a/squids/t/CommonConfig.pm b/squids/t/CommonConfig.pm
index 224441d..44535fa 100644
--- a/squids/t/CommonConfig.pm
+++ b/squids/t/CommonConfig.pm
@@ -17,7 +17,7 @@
our $__CODE_BASE;
if($hostname eq "gallium") {
# Running on Jenkins
- $__CODE_BASE = "/var/lib/jenkins/jobs/analytics-wikistats/workspace/squids";
+ $__CODE_BASE = $ENV{WORKSPACE}."/squids";
} elsif($hostname eq "stat1" && $ENV{HOME} eq "/home/ezachte") {
# Running on Erik's account on stat1
$__CODE_BASE = "/home/ezachte/wikistats/squids";
diff --git a/squids/t/Generate/Squid.pm b/squids/t/Generate/Squid.pm
index 1bce9e8..b63b645 100644
--- a/squids/t/Generate/Squid.pm
+++ b/squids/t/Generate/Squid.pm
@@ -224,7 +224,7 @@
referer_header =>
"http://en.wikipedia.org/wiki/Phil_of_the_Future";,
x_forwarded_for_header => "-",
user_agent_header =>
"Mozilla/5.0%20(Windows%20NT%206.1;%20WOW64;%20rv:15.0)%20Gecko/20100101%20Firefox/15.0.1",
- geocode => "US",
+ geocode => "--",
};
my $field_client_ip;
@@ -257,7 +257,11 @@
my $field_referer_header = $params->{referer_header} //
$default->{referer_header};
my $field_x_forwarded_for_header = $params->{x_forwarded_for_header} //
$default->{x_forwarded_for_header};
my $field_user_agent_header = $params->{user_agent_header} //
$default->{user_agent_header};
- my $field_geocode = $params->{geocode} //
$default->{geocode};
+
+ my $geocode = $params->{geocode} //
$default->{geocode};
+
+ $field_client_ip .= "|$geocode";
+
# Currently wikistats accepts 15-field entries, geocode field included (will
have to talk to Erik about fields
# 15 and 16 in the description above because we seem to be dropping those at
the 15-field-count-check)
@@ -276,7 +280,6 @@
$field_referer_header ,
$field_x_forwarded_for_header,
$field_user_agent_header ,
- $field_geocode ,
);
$self->{buffer} .= "$raw_logline\n";
--
To view, visit https://gerrit.wikimedia.org/r/62528
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9856f648a6e8013d264ef9c05d5a571d0703040f
Gerrit-PatchSet: 2
Gerrit-Project: analytics/wikistats
Gerrit-Branch: master
Gerrit-Owner: Stefan.petrea <[email protected]>
Gerrit-Reviewer: Diederik <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
