https://www.mediawiki.org/wiki/Special:Code/MediaWiki/115166
Revision: 115166
Author: a_engels
Date: 2012-05-09 15:39:56 +0000 (Wed, 09 May 2012)
Log Message:
-----------
Count:
* add counting of mobile devices (uses new file MobileDeviceTypes.csv)
* separate internal (from Wikimedia servers) as a separate category for country
info - not yet worked into report!
* recognize more spaces in mime type
* log lines that are too short or too long
Report:
* add mobile devices report
* add user agents report as csv
* base countryinfo-report on useragents instead of browser/operating systems
* avoid showing non-integer numbers through JavaScript
Modified Paths:
--------------
trunk/wikistats/squids/SquidCountArchive.pl
trunk/wikistats/squids/SquidCountArchiveProcessLogRecord.pm
trunk/wikistats/squids/SquidCountArchiveReadInput.pm
trunk/wikistats/squids/SquidCountArchiveWriteOutput.pm
trunk/wikistats/squids/SquidReportArchive.pl
Added Paths:
-----------
trunk/wikistats/squids/MobileDeviceTypes.csv
trunk/wikistats/squids/SquidCountArchiveConfig.pm
Added: trunk/wikistats/squids/MobileDeviceTypes.csv
===================================================================
--- trunk/wikistats/squids/MobileDeviceTypes.csv
(rev 0)
+++ trunk/wikistats/squids/MobileDeviceTypes.csv 2012-05-09 15:39:56 UTC
(rev 115166)
@@ -0,0 +1,89 @@
+iPod,iPod,iOS
+Googlebot-Mobile,Googlebot-Mobile,Other
+Kindle,Kindle,Android
+CFNetwork,iOS app,iOS
+Dalvik,Android app,Android
+WikipediaMobile,Wikimedia Android app,Android
+PlayBook,PlayBook,Blackberry
+Google Web Preview,Google Web Preview: Android,Other
+iPhone;,iPhone,iOS
+Android,Android,Android
+BREW,BREW,Other
+Windows Phone,Windows Phone,Windows
+Brew,BREW,Other
+Symbian,Symbian,Symbian
+LG,LG,Other
+iPad,iPad,iOS
+iPhone,iPhone,iOS
+PPC,Pocket PC,Windows
+Tablet PC,Tablet PC,Windows
+Windows CE,Windows CE,Windows
+Windows Mobile,Windows Mobile,Windows
+GINGERBREAD,Android,Android
+Silk-Accelerated,Kindle,Android
+Macintosh; U; Intel Mac OS,Android,Android
+BMP,BREW,Other
+HTC,HTC,Other
+PLAYSTATION 3,Playstation 3,Playstation
+PlayStation Portable,PlayStation Portable,Playstation
+Google WAP Proxy,Google WAP Proxy,Other
+BlackBerry,BlackBerry,Blackberry
+J2ME/MIDP; Opera Mini,unspecified Opera Mini,Other
+\(J2ME\); Opera Mini,unspecified Opera Mini,Other
+Nintendo 3DS,Nintendo 3DS,Nintendo
+ZuneWP7,Windows Mobile,Windows
+SymbianOS,SymbianOS,Symbian
+Alcatel,Alcatel,Other
+S60,Series 60,Symbian
+SymbOS,Symbian,Symbian
+MeeGo,MeeGo/Tizen,Other
+Series40,Series40,Symbian
+Nokia,Nokia,Symbian
+nokia,Nokia,Symbian
+N900,Nokia,Symbian
+Series 60,Series 60,Symbian
+PlayStation Vita,Playstation Vita,Playstation
+Nintendo DSi,Nintendo DSi,Nintendo
+Pantech,Pantech,Other
+PANTECH,Pantech,Other
+Nintendo Wii,Nintendo Wii,Nintendo
+Bada,Bada,Bada
+SAMSUNG,Samsung,Bada
+Pixi,Palm,Other
+wapedia,wapedia,Other
+Pre/,Palm,Other
+SonyEricsson,SonyEricsson,Other
+gt-s,Samsung,Bada
+maui,unspecified WAP,Other
+j2me/UC Browser,unspecified UC Browser,Other
+gt-i,Samsung,Bada
+gt-m,Samsung,Bada
+micromax,Micromax,Bada
+UNTRUSTED,unspecified Java,Other
+MOT-,Motorola,Other
+HUAWEI,Huawei,Other
+KWC,Kyocera,Other
+EBRD1101,Sony Reader,Other
+Motorola,Motorola,Other
+ASTRO36,Huawei,Other
+MAUI,unspecified WAP,Other
+PM-8200,Sanyo,Other
+NOKIA,NOKIA,Symbian
+UCWEB,unspecified UC Browser,Other
+Palm,Palm,Other
+Blazer,Palm,Other
+SoftBank,SoftBank,Other
+timewe.net,timewe WAP simulator,Other
+NF-Browser,3 Skypephone,Other
+AU-MIC/2.0,Samsung,Bada
+OPWV,Openwave,Other
+Sunrise,Sunrise,Other
+AUDIOVOX,Audiovox,Other
+Haier,Haier,Other
+WapOnWindows,WapOnWindows,Other
+portalmmm,Samsung,Bada
+KDDI,KDDI,Other
+KCI,KCI,Other
+MXit WebBot,MXit WebBot,Other
+DoCoMo,DoCoMo,Other
+Dorado,unspecified WAP,Other
Modified: trunk/wikistats/squids/SquidCountArchive.pl
===================================================================
--- trunk/wikistats/squids/SquidCountArchive.pl 2012-05-09 15:35:06 UTC (rev
115165)
+++ trunk/wikistats/squids/SquidCountArchive.pl 2012-05-09 15:39:56 UTC (rev
115166)
@@ -55,11 +55,12 @@
$time_start = time ;
$path_root = $job_runs_on_production_server ? $cfg_path_root_production :
$cfg_path_root_test ;
- $tags_wiki_mobile =
"CFNetwork|Dalvik|WikipediaMobile|Appcelerator|WiktionaryMobile" ;
+ $tags_wiki_mobile =
"CFNetwork|Dalvik|WikipediaMobile|Appcelerator|WiktionaryMobile|Wikipedia
Mobile" ;
- $tags_mobile = "Android|BlackBerry|Windows
CE|DoCoMo|iPad|iPod|iPhone|HipTop|Kindle|LGE|Linux
arm|MIDP|NetFront|Nintendo|Nokia|Obigo|Opera Mini|Opera
Mobi|Palm|Playstation|Samsung|SoftBank|SonyEricsson|SymbianOS|UP\.Browser|Vodafone|WAP|webOS|HTC|KDDI|FOMA|Polaris|Teleca|Silk|ZuneWP|HUAwei|Sunrise|AUDIOVOX|LG/U|AU-MIC|Motorola|portalmmm|Amoi|GINGERBREAD|Spice|lgtelecom|PlayBook|KYOCERA|Opera
Tablet|Windows
Phone|UNTRUSTED|Sensation|UCWEB|Nook|XV6975|EBRD1|Rhodium|UPG|Symbian|Pantech" ;
+ $tags_mobile = "Android|BlackBerry|Windows
CE|DoCoMo|iPad|iPod|iPhone|HipTop|Kindle|LGE|Linux
arm|MIDP|NetFront|Nintendo|Nokia|Obigo|Opera Mini|Opera
Mobi|Palm|Playstation|Samsung|SoftBank|SonyEricsson|Symbian|UP\.Browser|Vodafone|WAP|webOS|HTC[^P]|KDDI|FOMA|Polaris|Teleca|Silk|ZuneWP|HUAwei|Sunrise
XP|Sunrise/|AUDIOVOX|LG/U|AU-MIC|Motorola|portalmmm|Amoi|GINGERBREAD|Spice|lgtelecom|PlayBook|KYOCERA|Opera
Tablet|Windows
Phone|UNTRUSTED|Sensation|UCWEB|Nook|XV6975|EBRD1|Rhodium|UPG|Symbian|Pantech|MeeGo|Tizen"
;
$tags_tablet = "iPad|Android 3|SCH-I800|Kindle
Fire|Xoom|GT-P|Transformer|SC-01C|pandigital|SPH-P|STM803HC|K080|SGH-T849|CatNova|NookColor|M803HC|A1_|SGH-I987|Ideos
S7|SHW-M180|HomeManager|HTC_Flyer|PlayBook|Streak|Kobo
Touch|LG-V905R|MID7010|CT704|Silk|MID7024|ARCHM|Iconia|TT101|CT1002|;
A510|MID_Serials|ZiiO10|MID7015|001DL|MID Build|PM1152|RBK-490|Tablet|A100
Build|ViewPad|PMP3084|PG41200|; A500|A7EB|A80KSC" ;
$tags_mobile_upd = "March 2012" ;
+ &ReadMobileDeviceInfo ;
$pattern_url_pre = "(?:^|[a-zA-Z0-9-]+\\.)*?" ;
$pattern_url_post =
"\\.(?:biz|com|info|name|net|org|pro|aero|asia|cat|coop|edu|gov|int|jobs|mil|mobi|museum|tel|travel|arpa|[a-zA-Z0-9-]{2}|(?:com?|ne)\\.[a-zA-Z0-9-]{2})\$"
;
@@ -304,6 +305,7 @@
$file_csv_countries_timed = "public/SquidDataCountriesViewsTimed.csv" ; #
was SquidDataCountriesTimed2.csv
$file_csv_countries_saves = "public/SquidDataCountriesSaves.csv" ;
$file_csv_bots = "public/SquidDataCrawlers.csv" ;
+ $file_csv_devices = "public/SquidDataDevices.csv" ;
$file_csv_extensions = "public/SquidDataExtensions.csv" ;
$file_csv_googlebots = "public/SquidDataGoogleBots.csv" ;
$file_csv_images = "public/SquidDataImages.csv" ;
@@ -480,6 +482,8 @@
undef %countries_saves ;
undef %countries_timed ;
undef %countries_views ;
+ undef %country_info ;
+ undef %devices ;
undef %edit_submit_filtered ;
undef %engines ;
undef %exts ;
@@ -513,9 +517,8 @@
undef %statusses ;
undef %total_clients ;
undef %unrecognized_domains ;
+ undef %useragents ;
undef %wikis ;
- undef %useragents ;
- undef %country_info ;
# undef @files ;
};
Added: trunk/wikistats/squids/SquidCountArchiveConfig.pm
===================================================================
--- trunk/wikistats/squids/SquidCountArchiveConfig.pm
(rev 0)
+++ trunk/wikistats/squids/SquidCountArchiveConfig.pm 2012-05-09 15:39:56 UTC
(rev 115166)
@@ -0,0 +1,19 @@
+#!/usr/bin/perl
+
+ $developer = "engels" ;
+ $cfg_liblocation = "/a/squid/stats/dev_$developer/scripts" ;
+
+ $cfg_path_root_production = "/a/squid/stats/dev_$developer/csv" ;
+ $cfg_path_root_test = "w:/! perl/squids/archive/test" ; # Erik
+
+ $cfg_dir_in_production = "/a/squid/archive/sampled" ;
+ $cfg_dir_in_test = "?" ; # Erik
+
+ $cfg_logname = "sampled-1000.log" ;
+
+# set default arguments for test on local machine
+ $cfg_default_argv = "-d 2012/04/01-2012/04/30" ;
+
+ $cfg_file_test = "w:/! Perl/Squids/Archive/sampled-1000.log-20111016.txt" ;
# Erik
+ $cfg_test_maxlines = 4000000 ;
+
Modified: trunk/wikistats/squids/SquidCountArchiveProcessLogRecord.pm
===================================================================
--- trunk/wikistats/squids/SquidCountArchiveProcessLogRecord.pm 2012-05-09
15:35:06 UTC (rev 115165)
+++ trunk/wikistats/squids/SquidCountArchiveProcessLogRecord.pm 2012-05-09
15:39:56 UTC (rev 115166)
@@ -130,6 +130,8 @@
$country = $fields [14] ;
if (($country eq "") || ($country =~ /null/))
{ $country = "--" ; }
+ if (&isInternal($client_ip))
+ { $country = "-X" ; }
}
else
{
@@ -505,12 +507,12 @@
{ ($version = $agent2) =~ s/^.*?(Dalvik\/\d+\.?\d*).*$/Android: $1/o ; }
# WIKIPEDIA MOBILE
- elsif ($agent2 =~ /(Wiktionary|Wikipedia)Mobile(\/| )\d+/io)
+ elsif ($agent2 =~ /(Wiktionary|Wikipedia) ?Mobile(\/| )\d+/io)
{
if ($agent2 =~ /Android/io)
- { ($version = $agent2) =~ s/^.*((Wiktionary|Wikipedia)Mobile(\/|
)(\d|\.)+).*$/Android: $1 (WMF)/o ; }
+ { ($version = $agent2) =~ s/^.*((Wiktionary|Wikipedia) ?Mobile(\/|
)(\d|\.)+).*$/Android: $1 (WMF)/o ; }
else
- { ($version = $agent2) =~ s/^.*((Wiktionary|Wikipedia)Mobile(\/|
)(\d|\.)+).*$/iOS: $1 (WMF)/o ; }
+ { ($version = $agent2) =~ s/^.*((Wiktionary|Wikipedia) ?Mobile(\/|
)(\d|\.)+).*$/iOS: $1 (WMF)/o ; }
}
# ANDROID
@@ -698,8 +700,32 @@
$api =~ s/^.*action=([^\&]*)(\&.*)?$/$1/io ;
}
+ #create device info
+
+ if ($mobile =~ /[MWT]/ )
+ {
+ $device = "Unknown" ;
+ $devcat = "Other" ;
+ ($originalagent2 = $agent) =~ s/\%20/ /g ;
+ foreach $compared_device (@mobile_devices)
+ {
+ ( $devrecognition, $devtype, $devcategory ) = split( ',',
$compared_device ) ;
+ if ( $originalagent2 =~ /$devrecognition/ )
+ {
+ ($device = $devtype) =~ s/^\s+// ;
+ ($devcat = $devcategory) =~ s/\s+$// ;
+ last ;
+ }
+ }
+ if ($device eq "Unknown")
+ { print "Device $originalagent2 unknown.\n" ; }
+ $devices { "$device,$devcat" } += $count_event ;
+ }
+
#create useragents
$browsertype = $mobile ;
+ $maindomain = $domain ;
+ $maindomain =~ s/:.*//io ;
if ($bot)
{ $browsertype = 'B' ; }
elsif ($browsertype eq "-")
@@ -722,7 +748,7 @@
else
{ $browsertype = 'i' ; }
}
- $useragents {"$browsertype,$domain,$mimecat,$api"} += $count_event;
+ $useragents {"$browsertype,$maindomain,$mimecat,$api"} += $count_event;
#create countryinfo
if ((! $bot) && ($agent ne "-"))
@@ -734,6 +760,11 @@
$country_info {"M,$country,$mobile"} += $count_event;
$country_info {"B,$country,$browser"} += $count_event;
$country_info {"O,$country,$osshort"} += $count_event;
+ if ($mobile =~ /[MWT]/ )
+ {
+ $country_info {"D,$country,$devtype"} += $count_event;
+ }
+ $country_info {"C,$country,$browsertype;$maindomain;$mimecat;$api"} +=
$count_event;
}
# if ($domain_mobile)
@@ -1221,6 +1252,17 @@
return ($service, $agent) ;
}
+sub IsInternal
+{
+ # True iff the ip is from Wikimedia itself
+ my $address = shift ;
+ if ( $address =~ /^208\.80\.15[2345]/ )
+ { return $true ; }
+ if ( $address =~ /^91.198.174/ )
+ { return $true ; }
+ return $false ;
+}
+
sub MatchIpRange
{
my $address = shift ;
@@ -1352,4 +1394,23 @@
}
}
+sub ReadMobileDeviceInfo
+{
+ @mobile_devices = () ;
+ my $mobile_device_csv = 'MobileDeviceTypes.csv' ;
+ if (! -e $mobile_device_csv)
+ {
+ print "No mobile devices file found; all mobile devices will be counted as
unknown\n" ;
+ return
+ }
+ open CSV_MOBILE_DEVICES, '<', $mobile_device_csv ;
+ while ($line = <CSV_MOBILE_DEVICES>)
+ {
+ chomp $line ;
+ push( @mobile_devices, $line ) ;
+ print "Found mobile device $line\n" ;
+ }
+ close CSV_MOBILE_DEVICES ;
+}
+
1;
Modified: trunk/wikistats/squids/SquidCountArchiveReadInput.pm
===================================================================
--- trunk/wikistats/squids/SquidCountArchiveReadInput.pm 2012-05-09
15:35:06 UTC (rev 115165)
+++ trunk/wikistats/squids/SquidCountArchiveReadInput.pm 2012-05-09
15:39:56 UTC (rev 115166)
@@ -201,7 +201,7 @@
if ($test)
{ $line .= ' XX' ; }
- $line =~ s/x-www-form-urlencoded;
charset=UTF-8/x-www-form-urlencoded;%20charset=UTF-8/ ; # log lines are space
delimited, other spaces should be encoded
+ $line =~ s/; charset/;%20charset/ ; # log lines are space delimited,
other spaces should be encoded
@fields = split (' ', $line) ;
# next if $line =~ /upload/ ;
@@ -240,6 +240,7 @@
$fields_too_few ++ ;
print "invalid field count " . $#fields . "\n" ;
print ERR $#fields . " fields: \"$line\"\n" ;
+ &Log ("Short line: $line\n") ;
next ;
}
@@ -249,6 +250,7 @@
$fields_too_many ++ ;
print "invalid field count " . $#fields . "\n" ;
print ERR $#fields . " fields: \"$line\"\n" ;
+ &Log ("Long line: $line\n") ;
next ;
}
Modified: trunk/wikistats/squids/SquidCountArchiveWriteOutput.pm
===================================================================
--- trunk/wikistats/squids/SquidCountArchiveWriteOutput.pm 2012-05-09
15:35:06 UTC (rev 115165)
+++ trunk/wikistats/squids/SquidCountArchiveWriteOutput.pm 2012-05-09
15:39:56 UTC (rev 115166)
@@ -223,6 +223,7 @@
open OUT_REFERERS, '>', $file_out_referers ;
open CSV_CLIENTS_BY_WIKI, '>', $file_csv_clients_by_wiki ;
open CSV_AGENTS, '>', $file_csv_agents ;
+ open CSV_DEVICES, '>', $file_csv_devices ;
open CSV_USERAGENTS, '>', $file_csv_useragents ;
open CSV_COUNTRIES_INFO, '>', $file_csv_countries_info ;
@@ -250,6 +251,7 @@
print OUT_REFERERS $comment ;
print CSV_CLIENTS_BY_WIKI $comment ;
print CSV_AGENTS $comment ;
+ print CSV_DEVICES $comment ;
print CSV_USERAGENTS $comment ;
print CSV_COUNTRIES_INFO $comment ;
@@ -452,8 +454,15 @@
print CSV_GOOGLEBOTS "$date,$iprange,${google_bot_hits{$key}}\n" ;
}
+ # CSV_DEVICES ;
+ foreach $key (keys %devices)
+ {
+ my $count = $devices{$key} ;
+ print CSV_DEVICES "$key,$count\n" ;
+ }
+
# CSV_USERAGENTS ;
- foreach $key (%useragents)
+ foreach $key (keys %useragents)
{
my $count = $useragents{$key} ;
print CSV_USERAGENTS "$key, $count\n" ;
@@ -745,7 +754,7 @@
print CSV_AGENTS "$key,$count\n" ;
}
- print CSV_COUNTRIES_INFO "# pos 1: M = mobile/non-mobile status, O = opsys,
B = browser/client\n" ;
+ print CSV_COUNTRIES_INFO "# pos 1: M = mobile/non-mobile status, O = opsys,
B = browser/client, C = user agent category\n" ;
print CSV_COUNTRIES_INFO ":type,country,value,count\n" ;
foreach $key (sort keys %country_info)
@@ -897,8 +906,8 @@
{
if ($key !~ /$criterion/)
{
- if (($trace_count++ < 20) && ($criteria =~ /Linux/))
- { print "key $key criterion $criterion FALSE\n" ; }
+ #if (($trace_count++ < 20) && ($criteria =~ /Linux/))
+ #{ print "key $key criterion $criterion FALSE\n" ; }
$match = $false ;
last ;
}
Modified: trunk/wikistats/squids/SquidReportArchive.pl
===================================================================
--- trunk/wikistats/squids/SquidReportArchive.pl 2012-05-09 15:35:06 UTC
(rev 115165)
+++ trunk/wikistats/squids/SquidReportArchive.pl 2012-05-09 15:39:56 UTC
(rev 115166)
@@ -147,12 +147,13 @@
$file_html_scripts = "SquidReportScripts.htm" ;
$file_html_skins = "SquidReportSkins.htm" ;
$file_html_user_agents = "SquidReportUserAgents.htm" ;
+ $file_html_devices = "SquidReportDevices.htm" ;
+ $file_csv_user_agents_out = "SquidReportUserAgents.csv" ;
$file_html_requests = "SquidReportRequests.htm" ;
$file_html_google = "SquidReportGoogle.htm" ;
$file_html_clients = "SquidReportClients.htm" ;
$file_html_clients_html = "SquidReportClientsHtmlOnly.htm" ;
$file_html_countries_info = "SquidReportCountryData.htm" ;
-
$file_csv_user_agents = "SquidReportUserAgents.csv" ;
# names till 2010-07-01
#
@@ -184,6 +185,7 @@
$file_csv_indexphp = "public/SquidDataIndexPhp.csv" ;
$file_csv_browser_languages = "public/SquidDataLanguages.csv" ;
$file_csv_user_agents = "public/SquidDataUserAgents.csv" ;
+ $file_csv_devices = "public/SquidDataDevices.csv" ;
$file_csv_countries_info= "public/SquidDataCountriesInfo.csv" ;
$file_csv_countries_languages_visited = "SquidDataCountriesViews.csv" ;
@@ -244,6 +246,7 @@
&ReadInputMethods ;
&ReadInputMimeTypes ;
&ReadInputOpSys ;
+ &ReadInputDevices ;
&ReadInputOrigins ;
&ReadInputScripts ;
&ReadInputGoogle ;
@@ -326,6 +329,7 @@
&WriteReportMethods ;
&WriteReportMimeTypes ;
&WriteReportOpSys ;
+ &WriteReportDevices ;
&WriteReportOrigins ;
&WriteReportScripts ;
&WriteReportGoogle ;
@@ -576,7 +580,9 @@
$dummy_crawlers = "<font color=#000060>Crawlers</font>" ;
$dummy_opsys = "<font color=#000060>Op.Sys.</font>" ;
$dummy_browsers = "<font color=#000060>Browsers</font>" ;
+ $dummy_devices = "<font color=#000060>Devices</font>" ;
$dummy_google = "<font color=#000060>Google</font>" ;
+ $dummy_countries = "<font color=#000060>Country data</font>" ;
$link_requests = "Requests <a href='$file_html_requests'>by
destination</a> or " ;
$link_origins = "<a href='$file_html_origins'>by origin</a>" ;
@@ -587,8 +593,9 @@
$link_crawlers = "<a href='$file_html_crawlers'>Crawlers</a>" ;
$link_opsys = "<a href='$file_html_opsys'>Op.Sys.</a>" ;
$link_browsers = "<a href='$file_html_clients'>Browsers</a>" ;
+ $link_devices = "<a href='$file_html_devices'>Mobile devices</a>" ;
$link_google = "<a href='$file_html_google'>Google</a>" ;
- $link_countries = "<a href='$file_html_countries_info'>data by
country</a>" ;
+ $link_countries = "<a href='$file_html_countries_info'>Country data</a>" ;
}
sub ReadCountryCodes
@@ -1430,6 +1437,7 @@
$country_names {$country_code} = $country_name ;
$country_codes_all {"$country_name|$country_code"} ++ ;
}
+
}
sub ReadInputCountriesMeta
@@ -1972,28 +1980,36 @@
}
+sub ReadInputDevices
+{
+ &Log ("ReadInputDevices\n") ;
+
+ my $file_csv = "$path_process/$file_csv_devices" ;
+ if (! -e $file_csv)
+ { abort ("Function ReadInputDevices: file $file_csv not found!!!") ; }
+ open CSV_DEVICES, '<', $file_csv ;
+
+ while ($line = <CSV_DEVICES>)
+ {
+ next if $line =~ /^#/ ; # comments
+ next if $line =~ /^:/ ; # csv header (not a comment)
+
+ chomp ($line) ;
+
+ (my $device, my $category, my $count) = split (',', $line) ;
+ $devices { $device } += $count ;
+ $catfordevice { $device } = $category ;
+ $devicecats { $category } += $count ;
+ $alldevices += $count ;
+ }
+}
+
sub ReadInputCountriesInfo
{
&Log ("ReadInputCountriesInfo\n") ;
my $file_csv = "$path_process/$file_csv_countries_info" ;
if (! -e $file_csv)
{ abort ("Function ReadInputCountryInfo: file $file_csv not found!!!") ; }
- #$allcountrytotal = 0 ;
- #$countrytotal = { } ;
- #$allcountrybrowser = { } ;
- #$countrybrowser = { } ;
- #$allcountryos = { } ;
- #$countryos = { } ;
- #$allcountrymobile = 0 ;
- #$countrymobile = { } ;
- undef $allcountrytotal ;
- undef %countrytotal ;
- undef %allcountrybrowser ;
- undef %countrybrowser ;
- undef %allcountryos ;
- undef %countryos ;
- undef $allcountrymobile ;
- undef %countrymobile ;
open CSV_COUNTRIES_INFO, '<', $file_csv ;
while ($line = <CSV_COUNTRIES_INFO>)
{
@@ -2024,6 +2040,41 @@
$countryos { $country, $value } += $count ;
$allcountryos { $value } += $count ;
}
+ elsif ($type eq 'D')
+ {
+ $countrydevice { $country, $value } += $count ;
+ $countryalldevice { $country } += $count ;
+ $countrydevice { '.', $value } += $count ;
+ $countryalldevice { '.' } += $count ;
+ }
+ elsif ($type eq 'C')
+ {
+ print "Reporting $value\n" ;
+ my ($client, $domain, $mimecat, $api) = split (';', $value) ;
+ if ($domain =~ /^%/)
+ { $domain = 'M' ; }
+ elsif (substr($domain, 1, 1) eq 'w')
+ { $domain = 'W' ; }
+ else
+ { $domain = 'X' ; }
+ if ($client eq 'a' || $client eq 'i')
+ {
+ $client = 'W' ;
+ }
+ if ($mimecat eq 'page' && $domain ne 'X')
+ {
+ print "Counting it.\n" ;
+ $countrycount { $country } += $count ;
+ $countryua { '.', '.', '.' } += $count ;
+ $countryua { $country, '.', '.' } += $count ;
+ $countryua { '.', $client, '.' } += $count ;
+ $countryua { $country, $client, '.' } += $count ;
+ $countryua { '.', '.', $domain } += $count ;
+ $countryua { $country, '.', $domain } += $count ;
+ $countryua { '.', $client, $domain } += $count ;
+ $countryua { $country, $client, $domain } += $count ;
+ }
+ }
}
close CSV_COUNTRIES_INFO ;
}
@@ -2304,7 +2355,7 @@
$html =~ s/TITLE/Wikimedia Traffic Analysis Report - Browsers e.a./ ;
$html =~ s/HEADER/Wikimedia Traffic Analysis Report - Browsers e.a./ ;
$html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
- $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $dummy_browsers \/ $link_google/ ;
+ $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $dummy_browsers \/ $link_google \/ $link_countries/ ;
$html =~ s/X1000/⇒ <font color=#008000><b>all counts x
1000<\/b><\/font>.<br>/ ;
# test code, all counts from csv files are in thousands (from 1:1000 sampled
page file) and will be scaled x 1000
@@ -2987,7 +3038,7 @@
$html =~ s/TITLE/Wikimedia Traffic Analysis Report - Crawler requests/ ;
$html =~ s/HEADER/Wikimedia Traffic Analysis Report - Crawler requests/ ;
$html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
- $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $dummy_crawlers \/
$link_opsys \/ $dummy_browsers \/ $link_google/ ;
+ $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $dummy_crawlers \/
$link_opsys \/ $link_devices \/ $dummy_browsers \/ $link_google \/
$link_countries/ ;
$html =~ s/X1000/⇒ <font color=#008000><b>all counts x
1000<\/b><\/font>.<br>/ ;
$html .= "<table border=1>\n" ;
@@ -3163,7 +3214,7 @@
$html =~ s/TITLE/Wikimedia Traffic Analysis Report - Request Methods/ ;
$html =~ s/HEADER/Wikimedia Traffic Analysis Report - Request Methods/ ;
$html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
- $html =~ s/LINKS/$link_requests $link_origins \/ $dummy_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_browsers \/ $link_google/ ;
+ $html =~ s/LINKS/$link_requests $link_origins \/ $dummy_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_devices \/ $link_browsers \/ $link_google \/
$link_countries/ ;
$html =~ s/X1000/⇒ <font color=#008000><b>all counts x
1000<\/b><\/font>.<br>/ ;
$html .= "<table border=0>\n" ;
@@ -3246,7 +3297,7 @@
$html =~ s/HEADER/Wikimedia Traffic Analysis Report - Requests by
destination/ ;
$html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
$html =~ s/NOTES/<br> This report shows where requests are sent to.
Report 'Requests by origin' shows where requests come from.<br> Those
numbers bear no direct relation.<br>/ ;
- $html =~ s/LINKS/$dummy_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_browsers \/ $link_google/ ;
+ $html =~ s/LINKS/$dummy_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_devices \/ $link_browsers \/ $link_google \/
$link_countries/ ;
$html .= "<table border=1>\n" ;
$header1 = "<tr><th colspan=2 class=l><small>x 1000</small></th><th
colspan=2 class=c>Totals</th><th class=c><font
color=#008000>Pages</font></th><th colspan=3 class=c><font
color=#900000>Images</font></th><th colspan=99 class=c>Other</th></tr>\n" ;
@@ -3412,7 +3463,7 @@
$html =~ s/TITLE/Wikimedia Traffic Analysis Report - Operating Systems/ ;
$html =~ s/HEADER/Wikimedia Traffic Analysis Report - Operating Systems/ ;
$html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
- $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$dummy_opsys \/ $link_browsers \/ $link_google/ ;
+ $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$dummy_opsys \/ $link_devices \/ $link_browsers \/ $link_google \/
$link_countries/ ;
$html =~ s/X1000/⇒ <font color=#008000><b>all counts x
1000<\/b><\/font>.<br>/ ;
$total_all2 = &FormatCount ($total_opsys_mobile + $total_opsys_non_mobile) ;
@@ -3629,7 +3680,7 @@
$html =~ s/TITLE/Wikimedia Traffic Analysis Report - Requests by origin/ ;
$html =~ s/HEADER/Wikimedia Traffic Analysis Report - Requests by origin/ ;
$html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
- $html =~ s/LINKS/$link_requests $dummy_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_browsers \/ $link_google/ ;
+ $html =~ s/LINKS/$link_requests $dummy_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_devices \/ $link_browsers \/ $link_google \/
$link_countries/ ;
$html =~ s/NOTES/<br> This report shows where requests come from.
Report 'Requests by destination' shows where requests are
serviced.<br> Those numbers bear no direct relation.<br>/ ;
$html .= "<table border=1>\n" ;
@@ -4245,7 +4296,7 @@
$html =~ s/TITLE/Wikimedia Traffic Analysis Report - Scripts/ ;
$html =~ s/HEADER/Wikimedia Traffic Analysis Report - Scripts/ ;
$html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
- $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$dummy_scripts \/ $link_skins \/ $link_crawlers \/ $link_opsys \/
$link_browsers \/ $link_google/ ;
+ $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$dummy_scripts \/ $link_skins \/ $link_crawlers \/ $link_opsys \/
$link_browsers \/ $link_google \/ $link_countries/ ;
$html =~ s/X1000/⇒ <font color=#008000><b>all counts x
1000<\/b><\/font>.<br>/ ;
$html .= "<table border=1>\n" ;
@@ -4424,7 +4475,7 @@
$html =~ s/TITLE/Wikimedia Traffic Analysis Report - Google requests/ ;
$html =~ s/HEADER/Wikimedia Traffic Analysis Report - Google requests/ ;
$html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
- $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_browsers \/ $dummy_google/ ;
+ $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_devices \/ $link_browsers \/ $dummy_google \/
$link_countries/ ;
$html =~ s/X1000/⇒ <font color=#008000><b>all counts x
1000<\/b><\/font>.<br>/ ;
$html .= "<table border=1 width=500 wrap>\n" ;
@@ -4747,7 +4798,7 @@
$html =~ s/TITLE/Wikimedia Traffic Analysis Report - Skins/ ;
$html =~ s/HEADER/Wikimedia Traffic Analysis Report - Skins/ ;
$html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
- $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $dummy_skins \/ $link_crawlers \/
$link_opsys \/ $link_browsers \/ $link_google/ ;
+ $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $dummy_skins \/ $link_crawlers \/
$link_opsys \/ $link_devices \/ $link_browsers \/ $link_google \/
$link_countries/ ;
$html =~ s/X1000/⇒ <font color=#008000><b>all counts x
1000<\/b><\/font>.<br>/ ;
$html .= "<table border=1>\n" ;
@@ -4886,6 +4937,58 @@
return $result ;
}
+sub UserAgentCsvLine
+{
+ my ($title, $code, $showperc) = @_ ;
+ my $result = $title,
+ $value = 1000 * ($countua {$code, 'M', 'page', '.'} + $countua {$code, 'W',
'page', '.'}) * $multiplier ;
+ $writevalue = sprintf("%.0f", $value) ;
+ $result .= ",$writevalue" ;
+ if ($showperc)
+ {
+ $perc = 0.1 * $value / $total_html ;
+ $perc = sprintf("%.2f", $perc) ;
+ $result .= ",$perc" ;
+ $perc = 100 * $countua {$code, 'M', 'page', '.'} * $multiplier /
$total_html ;
+ $perc = sprintf("%.2f", $perc) ;
+ $result .= ",$perc" ;
+ $perc = 100 * $countua {$code, 'W', 'page', '.'} * $multiplier /
$total_html ;
+ $perc = sprintf("%.2f", $perc) ;
+ $result .= ",$perc" ;
+ $perc = 100 * $countua {$code, '.', '.', 'opensearch'} * $multiplier /
$total_opensearch ;
+ $perc = sprintf("%.2f", $perc) ;
+ $result .= ",$perc" ;
+ }
+ else
+ {
+ $result .= ",,,," ;
+ }
+ $value = 1000 * $countua {$code, '.', '.', '.'} * $multiplier ;
+ $writevalue = sprintf("%.0f", $value) ;
+ $result .= ",$writevalue" ;
+ if ($showperc)
+ {
+ $perc = 0.1 * $value / $total_count ;
+ $perc = sprintf("%.2f", $perc) ;
+ $result .= ",$perc" ;
+ $perc = 100 * $countua {$code, 'M', '.', '.'} * $multiplier / $total_count
;
+ $perc = sprintf("%.2f", $perc) ;
+ $result .= ",$perc" ;
+ $perc = 100 * $countua {$code, 'W', '.', '.'} * $multiplier / $total_count
;
+ $perc = sprintf("%.2f", $perc) ;
+ $result .= ",$perc" ;
+ $perc = 100 * $countua {$code, 'X', '.', '.'} * $multiplier / $total_count
;
+ $perc = sprintf("%.2f", $perc) ;
+ $result .= ",$perc" ;
+ }
+ else
+ {
+ $result .= ",,,," ;
+ }
+ $result .= "\n" ;
+ return $result ;
+}
+
sub WriteReportUserAgents
{
&Log ("WriteReportUserAgents\n") ;
@@ -4894,15 +4997,14 @@
$altbgcolor = '#DDFFDD' ;
open FILE_HTML_USER_AGENTS, '>', "$path_reports/$file_html_user_agents" ;
- open FILE_CSV_USER_AGENTS, '>', "$path_reports/$file_csv_user_agents" ;
+ open FILE_CSV_USER_AGENTS, '>', "$path_reports/$file_csv_user_agents_out" ;
- $csv_out = "# user agents lay-out\n" ;
- $csv_out = "# pageviews total, pageviews mobile, pageviews main, opensearch,
all total, all mobile, all main, all other\n" ;
+ $csv_out = "# category, pageviews total, pageviews percentage, mobile, main,
search_estimate, all total, percentage, mobile, main, other\n" ;
$html = $header ;
$html =~ s/TITLE/Wikimedia Traffic Analysis Report - User Agent Overview/ ;
$html =~ s/HEADER/Wikimedia Traffic Analysis Report - User Agent Overview/ ;
$html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
- $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $dummy_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_browsers \/ $link_google/ ;
+ $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $dummy_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_devices \/ $link_browsers \/ $link_google \/
$link_countries/ ;
$html =~ s/X1000/⇒ <font color=#008000><b>all counts x
1000<\/b><\/font>.<br>/ ;
$html .= "<table border=1>\n" ;
@@ -4918,20 +5020,34 @@
$total_mobile_html = ($countua {'S', 'M', 'page', '.'} +
$countua {'S', 'W', 'page', '.'}) * $multiplier ;
$mobile_opensearch = $countua {'S', '.', '.', 'opensearch'} *
$multiplier ;
- $html .= &UserAgentLine("From all sources", '.', $false, $true, 0, 0) ;
+ $html .= &UserAgentLine("From all sources", 'Z', $false, $true, 0, 0) ;
+ $csv_out .= &UserAgentCsvLine("From all sources", '.', $false) ;
$html .= &UserAgentLine("From mobile devices", 'S', $true, $true, 1, 13) ;
+ $csv_out .= &UserAgentCsvLine("From mobile devices", 'S', $true) ;
$html .= &UserAgentLine("From mobile browsers", 'C', $true, $true, 2, 10) ;
+ $csv_out .= &UserAgentCsvLine("From mobile sources", 'C', $true) ;
$html .= &UserAgentLine("From tablet browsers", 'T', $true, $false, 3, 3) ;
+ $csv_out .= &UserAgentCsvLine("From tablet sources", 'T', $true) ;
$html .= &UserAgentLine("From other mobile browsers", 'M', $true, $false, 3,
0) ;
+ $csv_out .= &UserAgentCsvLine("From other mobile browsers", 'M', $true) ;
$html .= &UserAgentLine("Through WAP access", 'P', $true, $false, 3, 0) ;
+ $csv_out .= &UserAgentCsvLine("Through WAP access", 'P', $true) ;
$html .= &UserAgentLine("From mobile apps", 'Q', $true, $true, 2, 0) ;
+ $csv_out .= &UserAgentCsvLine("From mobile apps", 'Q', $true) ;
$html .= &UserAgentLine("Wikimedia Android apps", 'A', $true, $false, 3, 5) ;
+ $csv_out .= &UserAgentCsvLine("Wikimedia Android apps", 'A', $true) ;
$html .= &UserAgentLine("Other Android apps", 'a', $true, $false, 3, 0) ;
+ $csv_out .= &UserAgentCsvLine("Other Android apps", 'a', $true) ;
$html .= &UserAgentLine("Wikimedia iOS apps", 'I', $true, $false, 3, 0) ;
+ $csv_out .= &UserAgentCsvLine("Wikimedia iOS apps", 'I', $true) ;
$html .= &UserAgentLine("Other iOS apps", 'i', $true, $false, 3, 0) ;
+ $csv_out .= &UserAgentCsvLine("Other iOS apps", 'i', $true) ;
$html .= &UserAgentLine("Unspecified apps", 'W', $true, $false, 3, 0) ;
+ $csv_out .= &UserAgentCsvLine("Unspecified apps", 'W', $true) ;
$html .= &UserAgentLine("From non-mobile devices", 'N', $true, $true, 1, 0) ;
+ $csv_out .= &UserAgentCsvLine("From non-mobile devices", 'N', $true) ;
$html .= &UserAgentLine("From bots", 'B', $false, $false, 1, 0) ;
+ $csv_out .= &UserAgentCsvLine("From bots", 'B', $false) ;
$html .= "</table>\n" ;
$html .= "<p> </p>\n" ;
$html .= "<table border=1>\n" ;
@@ -4952,11 +5068,64 @@
$html .= $colophon_ae ;
print FILE_HTML_USER_AGENTS $html ;
+ print FILE_CSV_USER_AGENTS $csv_out ;
close FILE_HTML_USER_AGENTS ;
+ close FILE_CSV_USER_AGENTS ;
}
-sub WriteReportCountriesInfo
+sub SectionReportDevices
{
+ my ($section, $section_name) = @_ ;
+ $value = &ShowCount ( $devicecats { $section } * $multiplier ) ;
+ $perc = &ShowPerc ( 100 * $devicecats { $section } / $alldevices ) ;
+ my $result = "<tr><td class=lt rowspan=@@>$section_name</td><td class=rt
rowspan=@@>$value</td><td class=rt rowspan=@@>$perc</td>\n" ;
+ $devcounter = 0 ;
+ foreach $device (keys_sorted_by_value_num_desc %devices)
+ {
+ next if ( $catfordevice { $device } ne $section ) ;
+ $value = &ShowCount ( $devices { $device } * $multiplier ) ;
+ $perc = &ShowPerc ( 100 * $devices { $device } / $alldevices ) ;
+ if ($devcounter ne 0)
+ { $result .= "<tr>"; }
+ $result .= "<td class=lt>$device</td><td class=rt>$value</td><td
class=rt>$perc</td></tr>\n" ;
+ $devcounter++ ;
+ }
+ $result =~ s/@@/$devcounter/g ;
+ return $result ;
+}
+
+sub WriteReportDevices
+{
+ &Log ("WriteReportDevices\n") ;
+ $marker_color = '#8000FF' ;
+
+ open FILE_HTML_DEVICES, '>', "$path_reports/$file_html_devices" ;
+
+ $html = $header ;
+ $html =~ s/TITLE/Wikimedia Traffic Analysis Report - Mobile Device Types/ ;
+ $html =~ s/HEADER/Wikimedia Traffic Analysis Report - Mobile Device Types/ ;
+ $html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
+ $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $dummy_devices \/ $link_browsers \/ $link_google \/
$link_countries/ ;
+ $html =~ s/X1000/⇒ <font color=#008000><b>all counts x
1000<\/b><\/font>.<br>/ ;
+ $html .= "<table border=1>\n" ;
+ $html .= "<tr><th class=l>Device class</th><th class=c>Count</th><th
class=c>Percentage</th><th class=l>Device</th><th class=c>Count</th><th
class=c>Percentage</th></tr>" ;
+ $html .= &SectionReportDevices ( "iOS", "iOS" ) ;
+ $html .= &SectionReportDevices ( "Android", "Android" ) ;
+ $html .= &SectionReportDevices ( "Blackberry", "BlackBerry" ) ;
+ $html .= &SectionReportDevices ( "Windows", "Windows Mobile" ) ;
+ $html .= &SectionReportDevices ( "Symbian", "Symbian/other Nokia" ) ;
+ $html .= &SectionReportDevices ( "Bada", "Bada/other Samsung" ) ;
+ $html .= &SectionReportDevices ( "Playstation", "Playstation" ) ;
+ $html .= &SectionReportDevices ( "Nintendo", "Nintendo" ) ;
+ $html .= &SectionReportDevices ( "Other", "Other/Unknown" ) ;
+ $html .= "</table>\n" ;
+ $html .= $colophon_ae ;
+ print FILE_HTML_DEVICES $html ;
+ close FILE_HTML_DEVICES ;
+}
+
+sub WriteReportCountriesInfoOld
+{
&Log ("WriteReportCountriesInfo\n") ;
open FILE_HTML_COUNTRIES_INFO, '>',
"$path_reports/$file_html_countries_info" ;
@@ -4964,7 +5133,7 @@
$html =~ s/TITLE/Wikimedia Traffic Analysis Report - Data per Country/ ;
$html =~ s/HEADER/Wikimedia Traffic Analysis Report - Data per Country/ ;
$html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
- $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_browsers \/ $link_google/ ;
+ $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_devices \/ $link_browsers \/ $link_google \/
$dummy_countries/ ;
$html =~ s/X1000/⇒ <font color=#008000><b>all counts x
1000<\/b><\/font>.<br>/ ;
$html .= "<table border=1 width=800>\n" ;
@@ -5181,6 +5350,173 @@
close FILE_HTML_COUNTRIES_INFO ;
}
+sub ReportLineCountriesInfoGlobal
+{
+ my ($country, $countryshow) = @_ ;
+ my $result = "<tr><th class=l>$countryshow</th>" ;
+ $result .= &UserAgentField( $countryua { $country, '.', '.' } , $false ) ;
+ $rowvalue = $countryua { $country, '.', '.' } * $multiplier ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'I', '.' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'A', '.' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'W', '.' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'T', '.' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'M', '.' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'P', '.' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'N', '.' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'I', 'M' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'A', 'M' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'W', 'M' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'T', 'M' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'M', 'M' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'P', 'M' },
$rowvalue, $true, $false ) ;
+ $result .= &UserAgentFieldPerc( $countryua { $country, 'N', 'M' },
$rowvalue, $true, $false ) ;
+ $result .= "</tr>\n" ;
+ return $result ;
+}
+
+sub ReportLineCountriesInfoDevice
+{
+ my ($country, $countryshow) = @_ ;
+ my $result = "<tr><th class=l>$countryshow</th>" ;
+ foreach $category (@devicecategories)
+ {
+ $result .= &UserAgentFieldPerc( $countrydevice { $country, $category },
$countryalldevice { $country }, $true, $false ) ;
+ }
+ $result .= "</tr>\n" ;
+ return $result ;
+}
+
+sub WriteReportCountriesInfo
+{
+ &Log ("WriteReportCountriesInfo\n") ;
+ open FILE_HTML_COUNTRIES_INFO, '>',
"$path_reports/$file_html_countries_info" ;
+
+ my $html = $header ;
+ $html =~ s/TITLE/Wikimedia Traffic Analysis Report - Page view breakdown per
Country/ ;
+ $html =~ s/HEADER/Wikimedia Traffic Analysis Report - Page view breakdown
per Country/ ;
+ $html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ;
+ $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/
$link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/
$link_opsys \/ $link_devices \/ $link_browsers \/ $link_google \/
$dummy_countries/ ;
+ $html =~ s/X1000/⇒ <font color=#008000><b>all counts x
1000<\/b><\/font>.<br>/ ;
+
+ $html .= "<table border=1 width=800>\n" ;
+ $html .= "<tr><th class=c>Page views by country</th><th class=c
colspan='8'>All pageviews</th><th class=c colspan='7'>To Mobile
site</th></tr>\n" ;
+ $html .= "<tr><th class=c rowspan='2'>Country</th><th class=c
rowspan='2'>Total views</th><th class=c colspan='2'>Wikimedia apps</colspan><th
class=c rowspan='2'>Other apps</th><th class=c colspan=3>mobile browsers</c><th
class=c rowspan='2'>Non-mobile</th><th class=c colspan='2'>Wikimedia
apps</colspan><th class=c rowspan='2'>Other apps</th><th class=c
colspan=3>mobile browsers</c><th class=c rowspan='2'>Non-mobile</th></tr>\n" ;
+ $html .= "<tr><th class=c>iOS</th><th class=c>Android</th><th
class=c>Tablets</th><th class=c>Other</th><th class=c>WAP</th><th
class=c>iOS</th><th class=c>Android</th><th class=c>Tablets</th><th
class=c>Other</th><th class=c>WAP</th></tr>\n" ;
+ $html .= &ReportLineCountriesInfoGlobal( '.', '<b>All countries</b>' ) ;
+ foreach $country (keys_sorted_by_value_num_desc %countrycount)
+ {
+ my $region = $region_codes { $country} ;
+ $countryua {"reg" . $region, '.', '.' } += $countryua {$country, '.', '.'
} ;
+ $countryua {"reg" . $region, '.', '.' } += $countryua {$country, '.', '.'
} ;
+ $countryua {"reg" . $region, 'I', '.' } += $countryua {$country, 'I', '.'
} ;
+ $countryua {"reg" . $region, 'A', '.' } += $countryua {$country, 'A', '.'
} ;
+ $countryua {"reg" . $region, 'W', '.' } += $countryua {$country, 'W', '.'
} ;
+ $countryua {"reg" . $region, 'T', '.' } += $countryua {$country, 'T', '.'
} ;
+ $countryua {"reg" . $region, 'M', '.' } += $countryua {$country, 'M', '.'
} ;
+ $countryua {"reg" . $region, 'P', '.' } += $countryua {$country, 'P', '.'
} ;
+ $countryua {"reg" . $region, 'N', '.' } += $countryua {$country, 'N', '.'
} ;
+ $countryua {"reg" . $region, '.', 'M' } += $countryua {$country, '.', 'M'
} ;
+ $countryua {"reg" . $region, 'I', 'M' } += $countryua {$country, 'I', 'M'
} ;
+ $countryua {"reg" . $region, 'A', 'M' } += $countryua {$country, 'A', 'M'
} ;
+ $countryua {"reg" . $region, 'W', 'M' } += $countryua {$country, 'W', 'M'
} ;
+ $countryua {"reg" . $region, 'T', 'M' } += $countryua {$country, 'T', 'M'
} ;
+ $countryua {"reg" . $region, 'M', 'M' } += $countryua {$country, 'M', 'M'
} ;
+ $countryua {"reg" . $region, 'P', 'M' } += $countryua {$country, 'P', 'M'
} ;
+ $countryua {"reg" . $region, 'N', 'M' } += $countryua {$country, 'N', 'M'
} ;
+ }
+ $html .= &ReportLineCountriesInfoGlobal( 'regAF', '<b>Africa</b>' ) ;
+ $html .= &ReportLineCountriesInfoGlobal( 'regAS', '<b>Asia</b>' ) ;
+ $html .= &ReportLineCountriesInfoGlobal( 'regAU', '<b>Australia</b>' ) ;
+ $html .= &ReportLineCountriesInfoGlobal( 'regEU', '<b>Europe</b>' ) ;
+ $html .= &ReportLineCountriesInfoGlobal( 'regCA', '<b>Central America</b>' )
;
+ $html .= &ReportLineCountriesInfoGlobal( 'regNA', '<b>North America</b>' ) ;
+ $html .= &ReportLineCountriesInfoGlobal( 'regSA', '<b>South America</b>' ) ;
+ $html .= &ReportLineCountriesInfoGlobal( 'regOC', '<b>Oceania</b>' ) ;
+ $html .= "<tr><td colspan='16'> </td></tr>\n" ;
+ $countrycounter = 0 ;
+ foreach $country (keys_sorted_by_value_num_desc %countrycount)
+ {
+ next if ($country =~ /reg/ ) ;
+ if ($country eq '--' || $countrycounter eq 100)
+ {
+ $countryua {'other', '.', '.' } += $countryua {$country, '.', '.' } ;
+ $countryua {'other', '.', '.' } += $countryua {$country, '.', '.' } ;
+ $countryua {'other', 'I', '.' } += $countryua {$country, 'I', '.' } ;
+ $countryua {'other', 'A', '.' } += $countryua {$country, 'A', '.' } ;
+ $countryua {'other', 'W', '.' } += $countryua {$country, 'W', '.' } ;
+ $countryua {'other', 'T', '.' } += $countryua {$country, 'T', '.' } ;
+ $countryua {'other', 'M', '.' } += $countryua {$country, 'M', '.' } ;
+ $countryua {'other', 'P', '.' } += $countryua {$country, 'P', '.' } ;
+ $countryua {'other', 'N', '.' } += $countryua {$country, 'N', '.' } ;
+ $countryua {'other', '.', 'M' } += $countryua {$country, '.', 'M' } ;
+ $countryua {'other', 'I', 'M' } += $countryua {$country, 'I', 'M' } ;
+ $countryua {'other', 'A', 'M' } += $countryua {$country, 'A', 'M' } ;
+ $countryua {'other', 'W', 'M' } += $countryua {$country, 'W', 'M' } ;
+ $countryua {'other', 'T', 'M' } += $countryua {$country, 'T', 'M' } ;
+ $countryua {'other', 'M', 'M' } += $countryua {$country, 'M', 'M' } ;
+ $countryua {'other', 'P', 'M' } += $countryua {$country, 'P', 'M' } ;
+ $countryua {'other', 'N', 'M' } += $countryua {$country, 'N', 'M' } ;
+ }
+ else
+ {
+ $countrycounter += 1;
+ $html .= &ReportLineCountriesInfoGlobal( $country, $country_codes
{$country} ) ;
+ }
+ }
+ $html .= &ReportLineCountriesInfoGlobal( 'other', 'Other/Unknown') ;
+ $html .= "</table>\n" ;
+ $html .= $colophon_ae ;
+ print FILE_HTML_COUNTRIES_INFO $html ;
+ close FILE_HTML_COUNTRIES_INFO ;
+}
+
+sub AddLater
+{
+ @devicecategories = ( "iOS", "Android", "Blackberry", "Windows", "Symbian",
"Bada", "Playstation", "Nintendo" ) ;
+
+ $html .= "<table border=1>\n" ;
+ $html .= "<tr><th class=c>Mobile devices by country</th><th
class=c>Android</th><th class=c>iOS</th><th class=c>BlackBerry</th><th
class=c>Windows<br/>Mobile</th><th class=c>Symbian<br/>Nokia</th><th
class=c>Bada<br/>Samsung</th><th class=c>Playstation<br/>Nintendo</th><th
class=c>Other<br/>Unknown<th></tr>\n" ;
+ foreach $country (keys_sorted_by_value_num_desc %countryalldevice)
+ {
+ my $region = "reg" . $region_codes { $country} ;
+ foreach $category (@devicecategories)
+ { $countrydevice { $region, $category } += $countrydevice { $country,
$category } ; }
+ $countryalldevice { $region } += $countryalldevice { $country } ;
+ }
+ $html .= &ReportLineCountriesInfoDevice( '.', '<b>All countries</b>' ) ;
+ $html .= &ReportLineCountriesInfoDevice( 'regAF', '<b>Africa</b>' ) ;
+ $html .= &ReportLineCountriesInfoDevice( 'regAS', '<b>Asia</b>' ) ;
+ $html .= &ReportLineCountriesInfoDevice( 'regAU', '<b>Australia</b>' ) ;
+ $html .= &ReportLineCountriesInfoDevice( 'regEU', '<b>Europe</b>' ) ;
+ $html .= &ReportLineCountriesInfoDevice( 'regCA', '<b>Central America</b>' )
;
+ $html .= &ReportLineCountriesInfoDevice( 'regNA', '<b>North America</b>' ) ;
+ $html .= &ReportLineCountriesInfoDevice( 'regSA', '<b>South America</b>' ) ;
+ $html .= &ReportLineCountriesInfoDevice( 'regOC', '<b>Oceania</b>' ) ;
+ $html .= "<tr><td colspan='" . (scalar(@devicecategories) + 1) .
"'> </td></tr>\n" ;
+ $countrycounter = 0 ;
+
+ foreach $country (keys_sorted_by_value_num_desc %countryalldevice)
+ {
+ next if ($country =~ /reg/ ) ;
+ next if ($country eq '.') ;
+ if ($country eq '--' || $countrycounter eq 50)
+ {
+ foreach $category (@devicecategories)
+ { $countrydevice { 'other', $category } += $countrydevice { $country,
$category } ; }
+ $countryalldevice { 'other' } += $countryalldevice { $country } ;
+ }
+ else
+ {
+ $countrycounter += 1;
+ $html .= &ReportLineCountriesInfoDevice( $country, $country_codes
{$country} ) ;
+ }
+ }
+
+ $html .= &ReportLineCountriesInfoDevice( 'other', 'Other/Unknown') ;
+ $html .= "</table>\n" ;
+
+}
+
sub WriteCsvGoogleBots
{
&Log ("WriteCsvGoogleBots\n") ;
@@ -6630,7 +6966,7 @@
else
{
if ($num =~ /^[\d\.]+$/) # numeric string
- { $num *= 1000 ; }
+ { $num = ceil( 1000 * $num - 0.5) ; }
if ($num =~ /\D/) # contains non-digit ? enclose in double quotes
{ $num ="\"$num\"" ; }
@@ -7516,8 +7852,10 @@
{ count = (Math.round (count/100000) / 10) + nbsp + char_million ; }
else if (count >= 10000)
{ count = Math.round (count/1000) + nbsp + char_thousand ; }
- else if (count >= 1000)
+ else if (count >= 999)
{ count = (Math.round (count/100) / 10) + nbsp + char_thousand ; }
+ else
+ { count = Math.round (count) ; }
count += '' ; // make string
count = count.replace ($regexp_from4,$regexp_to4) ;
}
_______________________________________________
MediaWiki-CVS mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-cvs
