Erik Zachte has submitted this change and it was merged. Change subject: Added links for New mobile pageviews report ......................................................................
Added links for New mobile pageviews report * Added links to New mobile pageviews report in SquidReportArchive.pl * Added directories and script to run Wikistats * New mobile pageviews report will be rsync-ed to http://stats.wikimedia.org/wikimedia/squids/ as agreed with Erik in e-mail on 17 April 2013 * fixed points from April 22 e-mail * fixed points in review of patchset 5 Change-Id: Ica902912862af002278a9b7b95803415806a550b --- M dumps/bash/pageviews_monthly_sp.sh M squids/.gitignore A squids/basic-run.sh A squids/csv_basic/blank.sh A squids/logs_basic/blank.sh M squids/perl/SquidReportArchive.pl A squids/reports_basic/blank.sh 7 files changed, 114 insertions(+), 55 deletions(-) Approvals: Erik Zachte: Verified; Looks good to me, approved jenkins-bot: Verified diff --git a/dumps/bash/pageviews_monthly_sp.sh b/dumps/bash/pageviews_monthly_sp.sh index 28f77c6..7ca77e6 100755 --- a/dumps/bash/pageviews_monthly_sp.sh +++ b/dumps/bash/pageviews_monthly_sp.sh @@ -26,6 +26,16 @@ perl WikiReports.pl -v m -m wp -l en -q -i $csv/csv_sp/ -o $out/out_sp -n | tee -a $report | cat perl WikiReports.pl -v m -m wp -l en -q -i $csv/csv_sp/ -o $out/out_sp | tee -a $report | cat -rsync -av $out/out_sp/EN/TablesPageViewsMonthlySquidsMobile.htm $htdocs/EN/draft/TablesPageViewsMonthlySquidsMobile.htm -rsync -av $out/out_sp/EN/TablesPageViewsMonthlySquidsOriginalMobile.htm $htdocs/EN/draft/TablesPageViewsMonthlySquidsOriginalMobile.htm +################################################################# +# Cross-reference => The file name in the destination of rsync +# must be the same as the $file_html_new_mobile_pageviews inside +# SquidReportArchive.pl +################################################################# +#rsync -av $out/out_sp/EN/TablesPageViewsMonthlySquidsMobile.htm $htdocs/wikimedia/squids/TablesPageViewsMonthlySquidsMobile.htm + +########################################## +# Question => What is the Original file ? +# Why do we need it ? +########################################## +#rsync -av $out/out_sp/EN/TablesPageViewsMonthlySquidsOriginalMobile.htm $htdocs/wikimedia/squids/TablesPageViewsMonthlySquidsOriginalMobile.htm diff --git a/squids/.gitignore b/squids/.gitignore index 7becbcc..a25cb96 100644 --- a/squids/.gitignore +++ b/squids/.gitignore @@ -36,6 +36,16 @@ # and readme's !read* +# basic wikistats configuration +!conf-basic +!conf-basic/ +!csv_basic +!csv_basic/ +!reports_basic/ +!logs_basic/ + + + # advice Linus # use add -f to add few files in ignored folders # and now you're done - git won't ignore them, since explcitly you told git to track them diff --git a/squids/basic-run.sh b/squids/basic-run.sh new file mode 100755 index 0000000..f44ead3 --- /dev/null +++ b/squids/basic-run.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +#COUNT_PERIOD="2012/08/01-2012/08/31" +COUNT_PERIOD="2012/08/01-2012/08/01" +REPORT_PERIOD="2012-08" + +################################# +# CLEANUP +################################# + +cd csv_basic; ls | grep -v blank | xargs rm -rf; cd ..; +cd logs_basic; ls | grep -v blank | xargs rm -rf; cd ..; +cd reports_basic; ls | grep -v blank | xargs rm -rf; cd ..; +cd csv_basic; ln -s ../csv/meta ; cd ..; + +################################# +# RUNNING SquidCountArchive +################################# + + +perl -Iperl/ perl/SquidCountArchive.pl \ + -r conf-basic/SquidCountArchiveConfig.pm \ + -d $COUNT_PERIOD \ + -p 2>&1 > basic-count.log; + +################################# +# RUNNING SquidReportArchive +################################# + +perl -Iperl perl/SquidReportArchive.pl \ + -r conf-basic/SquidReportArchiveConfig.pm \ + -m $REPORT_PERIOD \ + -p 2>&1 > basic-report.log; + diff --git a/squids/csv_basic/blank.sh b/squids/csv_basic/blank.sh new file mode 100644 index 0000000..92f72ed --- /dev/null +++ b/squids/csv_basic/blank.sh @@ -0,0 +1 @@ +blank diff --git a/squids/logs_basic/blank.sh b/squids/logs_basic/blank.sh new file mode 100644 index 0000000..92f72ed --- /dev/null +++ b/squids/logs_basic/blank.sh @@ -0,0 +1 @@ +blank diff --git a/squids/perl/SquidReportArchive.pl b/squids/perl/SquidReportArchive.pl index 6b7b9ae..b0db7a9 100755 --- a/squids/perl/SquidReportArchive.pl +++ b/squids/perl/SquidReportArchive.pl @@ -148,28 +148,29 @@ $file_log = "SquidReportArchive.log" ; - $file_html_crawlers = "SquidReportCrawlers.htm" ; - $file_html_methods = "SquidReportMethods.htm" ; - $file_html_origins = "SquidReportOrigins.htm" ; - $file_html_opsys = "SquidReportOperatingSystems.htm" ; - $file_html_opsys_html = "SquidReportOperatingSystemsHtmlOnly.htm" ; - $file_html_scripts = "SquidReportScripts.htm" ; - $file_html_skins = "SquidReportSkins.htm" ; - $file_html_user_agents = "SquidReportUserAgents.htm" ; - $file_html_devices = "SquidReportDevices.htm" ; - $file_csv_user_agents_out = "SquidReportUserAgents.csv" ; - $file_html_requests = "SquidReportRequests.htm" ; - $file_html_google = "SquidReportGoogle.htm" ; - $file_html_clients = "SquidReportClients.htm" ; - $file_html_clients_html = "SquidReportClientsHtmlOnly.htm" ; - $file_html_countries_info = "SquidReportCountryData.htm" ; - $file_html_countries_browser = "SquidReportCountryBrowser.htm" ; - $file_html_countries_os = "SquidReportCountryOs.htm" ; - $file_html_countries_device = "SquidReportCountryDevice.htm" ; - $file_html_user_agents_timed = "SquidReportUserAgentsTimed.htm" ; - $file_html_browsers_timed = "SquidReportBrowsersTimed.htm" ; - $file_html_devices_timed = "SquidReportDevicesTimed.htm" ; - $file_csv_user_agents = "SquidReportUserAgents.csv" ; + $file_html_new_mobile_pageviews = "TablesPageViewsMonthlySquidsMobile.htm" ; + $file_html_crawlers = "SquidReportCrawlers.htm" ; + $file_html_methods = "SquidReportMethods.htm" ; + $file_html_origins = "SquidReportOrigins.htm" ; + $file_html_opsys = "SquidReportOperatingSystems.htm" ; + $file_html_opsys_html = "SquidReportOperatingSystemsHtmlOnly.htm" ; + $file_html_scripts = "SquidReportScripts.htm" ; + $file_html_skins = "SquidReportSkins.htm" ; + $file_html_user_agents = "SquidReportUserAgents.htm" ; + $file_html_devices = "SquidReportDevices.htm" ; + $file_csv_user_agents_out = "SquidReportUserAgents.csv" ; + $file_html_requests = "SquidReportRequests.htm" ; + $file_html_google = "SquidReportGoogle.htm" ; + $file_html_clients = "SquidReportClients.htm" ; + $file_html_clients_html = "SquidReportClientsHtmlOnly.htm" ; + $file_html_countries_info = "SquidReportCountryData.htm" ; + $file_html_countries_browser = "SquidReportCountryBrowser.htm" ; + $file_html_countries_os = "SquidReportCountryOs.htm" ; + $file_html_countries_device = "SquidReportCountryDevice.htm" ; + $file_html_user_agents_timed = "SquidReportUserAgentsTimed.htm" ; + $file_html_browsers_timed = "SquidReportBrowsersTimed.htm" ; + $file_html_devices_timed = "SquidReportDevicesTimed.htm" ; + $file_csv_user_agents = "SquidReportUserAgents.csv" ; # names till 2010-07-01 # @@ -628,30 +629,31 @@ $dummy_countries_os = "<font color=#000060>Operating systems</font>" ; $dummy_countries_devices = "<font color=#000060>Mobile devices</font>" ; - $link_requests = "Requests <a href='$file_html_requests'>by destination</a> or " ; - $link_origins = "<a href='$file_html_origins'>by origin</a>" ; - $link_methods = "<a href='$file_html_methods'>Methods</a>" ; - $link_scripts = "<a href='$file_html_scripts'>Scripts</a>" ; - $link_skins = "<a href='$file_html_skins'>Skins</a>" ; - $link_user_agents = "<a href='$file_html_user_agents'>User agents</a>" ; - $link_crawlers = "<a href='$file_html_crawlers'>Crawlers</a>" ; - $link_opsys = "<a href='$file_html_opsys'>Op.Sys.</a>" ; - $link_browsers = "<a href='$file_html_clients'>Browsers</a>" ; - $link_devices = "<a href='$file_html_devices'>Mobile devices</a>" ; - $link_google = "<a href='$file_html_google'>Google</a>" ; - $link_countries = "<a href='$file_html_countries_info'>Country data</a>" ; - $link_ua_trends = "<a href='$file_html_user_agents_timed'>Traffic trends</a>" ; - $link_countries_useragents = "<a href='$file_html_countries_info'>User agents</a>" ; - $link_countries_browsers = "<a href='$file_html_countries_browser'>Browsers</a>" ; - $link_countries_os = "<a href='$file_html_countries_os'>Operating systems</a>" ; - $link_countries_devices = "<a href='$file_html_countries_device'>Mobile devices</a>" ; - $link_countries_overview = "<a href='SquidReportPageViewsPerCountryOverview.htm'>Overview</a>" ; - $link_countries_projects = "<a href='SquidReportPageViewsPerCountryBreakdown.htm'>Projects</a>" ; - $link_countries_trends = "<a href='http://stats.wikimedia.org/wikimedia/squids/SquidReportPageViewsPerCountryTrends.htm'>Trends</a>" ; - $link_trends_useragents = "<a href='$file_html_user_agents_timed'>User agents</a>" ; - $link_trends_countries = "<a href='http://stats.wikimedia.org/wikimedia/squids/SquidReportPageViewsPerCountryTrends.htm'>Countries</a>" ; - $link_trends_browsers = "<a href='$file_html_browsers_timed'>Browsers</a>" ; - $link_trends_devices = "<a href='$file_html_devices_timed'>Devices</a>" ; + $link_requests = "Requests <a href='$file_html_requests'>by destination</a> or " ; + $link_new_mobile_pageviews = "<a href='$file_html_new_mobile_pageviews'>New Mobile Pageviews(from squid logs)</a>"; + $link_origins = "<a href='$file_html_origins'>by origin</a>" ; + $link_methods = "<a href='$file_html_methods'>Methods</a>" ; + $link_scripts = "<a href='$file_html_scripts'>Scripts</a>" ; + $link_skins = "<a href='$file_html_skins'>Skins</a>" ; + $link_user_agents = "<a href='$file_html_user_agents'>User agents</a>" ; + $link_crawlers = "<a href='$file_html_crawlers'>Crawlers</a>" ; + $link_opsys = "<a href='$file_html_opsys'>Op.Sys.</a>" ; + $link_browsers = "<a href='$file_html_clients'>Browsers</a>" ; + $link_devices = "<a href='$file_html_devices'>Mobile devices</a>" ; + $link_google = "<a href='$file_html_google'>Google</a>" ; + $link_countries = "<a href='$file_html_countries_info'>Country data</a>" ; + $link_ua_trends = "<a href='$file_html_user_agents_timed'>Traffic trends</a>" ; + $link_countries_useragents = "<a href='$file_html_countries_info'>User agents</a>" ; + $link_countries_browsers = "<a href='$file_html_countries_browser'>Browsers</a>" ; + $link_countries_os = "<a href='$file_html_countries_os'>Operating systems</a>" ; + $link_countries_devices = "<a href='$file_html_countries_device'>Mobile devices</a>" ; + $link_countries_overview = "<a href='SquidReportPageViewsPerCountryOverview.htm'>Overview</a>" ; + $link_countries_projects = "<a href='SquidReportPageViewsPerCountryBreakdown.htm'>Projects</a>" ; + $link_countries_trends = "<a href='http://stats.wikimedia.org/wikimedia/squids/SquidReportPageViewsPerCountryTrends.htm'>Trends</a>" ; + $link_trends_useragents = "<a href='$file_html_user_agents_timed'>User agents</a>" ; + $link_trends_countries = "<a href='http://stats.wikimedia.org/wikimedia/squids/SquidReportPageViewsPerCountryTrends.htm'>Countries</a>" ; + $link_trends_browsers = "<a href='$file_html_browsers_timed'>Browsers</a>" ; + $link_trends_devices = "<a href='$file_html_devices_timed'>Devices</a>" ; } sub ReadCountryCodes @@ -2596,7 +2598,7 @@ $html =~ s/TITLE/Wikimedia Traffic Analysis Report - Browsers e.a./ ; $html =~ s/HEADER/Wikimedia Traffic Analysis Report - Browsers e.a./ ; $html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ; - $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/ $link_opsys \/ $dummy_browsers \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; + $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/ $link_opsys \/ $dummy_browsers \/ $link_new_mobile_pageviews \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; $html =~ s/X1000/⇒ <font color=#008000><b>all counts x 1000<\/b><\/font>.<br>/ ; # test code, all counts from csv files are in thousands (from 1:1000 sampled page file) and will be scaled x 1000 @@ -3330,7 +3332,7 @@ $html =~ s/TITLE/Wikimedia Traffic Analysis Report - Crawler requests/ ; $html =~ s/HEADER/Wikimedia Traffic Analysis Report - Crawler requests/ ; $html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ; - $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $dummy_crawlers \/ $link_opsys \/ $link_devices \/ $dummy_browsers \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; + $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $dummy_crawlers \/ $link_opsys \/ $link_devices \/ $dummy_browsers \/ $link_new_mobile_pageviews \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; $html =~ s/X1000/⇒ <font color=#008000><b>all counts x 1000<\/b><\/font>.<br>/ ; $html .= "<table border=1>\n" ; @@ -3506,7 +3508,7 @@ $html =~ s/TITLE/Wikimedia Traffic Analysis Report - Request Methods/ ; $html =~ s/HEADER/Wikimedia Traffic Analysis Report - Request Methods/ ; $html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ; - $html =~ s/LINKS/$link_requests $link_origins \/ $dummy_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/ $link_opsys \/ $link_devices \/ $link_browsers \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; + $html =~ s/LINKS/$link_requests $link_origins \/ $dummy_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/ $link_opsys \/ $link_devices \/ $link_browsers \/ $link_new_mobile_pageviews \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; $html =~ s/X1000/⇒ <font color=#008000><b>all counts x 1000<\/b><\/font>.<br>/ ; $html .= "<table border=0>\n" ; @@ -3765,7 +3767,7 @@ $html =~ s/TITLE/Wikimedia Traffic Analysis Report - Operating Systems/ ; $html =~ s/HEADER/Wikimedia Traffic Analysis Report - Operating Systems/ ; $html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ; - $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/ $dummy_opsys \/ $link_devices \/ $link_browsers \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; + $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/ $dummy_opsys \/ $link_devices \/ $link_browsers \/ $link_new_mobile_pageviews \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; $html =~ s/X1000/⇒ <font color=#008000><b>all counts x 1000<\/b><\/font>.<br>/ ; $total_all2 = &FormatCount ($total_opsys_mobile + $total_opsys_non_mobile) ; @@ -3985,7 +3987,7 @@ $html =~ s/TITLE/Wikimedia Traffic Analysis Report - Requests by origin/ ; $html =~ s/HEADER/Wikimedia Traffic Analysis Report - Requests by origin/ ; $html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ; - $html =~ s/LINKS/$link_requests $dummy_origins \/ $link_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/ $link_opsys \/ $link_devices \/ $link_browsers \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; + $html =~ s/LINKS/$link_requests $dummy_origins \/ $link_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/ $link_opsys \/ $link_devices \/ $link_browsers \/ $link_new_mobile_pageviews \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; $html =~ s/NOTES/<br> This report shows where requests come from. Report 'Requests by destination' shows where requests are serviced.<br> Those numbers bear no direct relation.<br>/ ; $html .= "<table border=1>\n" ; @@ -4601,7 +4603,7 @@ $html =~ s/TITLE/Wikimedia Traffic Analysis Report - Scripts/ ; $html =~ s/HEADER/Wikimedia Traffic Analysis Report - Scripts/ ; $html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ; - $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/ $dummy_scripts \/ $link_skins \/ $link_crawlers \/ $link_opsys \/ $link_browsers \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; + $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/ $dummy_scripts \/ $link_skins \/ $link_crawlers \/ $link_opsys \/ $link_browsers \/ $link_new_mobile_pageviews \/ $link_new_mobile_pageviews \/ $link_google \/ $link_countries \/ $link_ua_trends/ ; $html =~ s/X1000/⇒ <font color=#008000><b>all counts x 1000<\/b><\/font>.<br>/ ; $html .= "<table border=1>\n" ; @@ -4780,7 +4782,7 @@ $html =~ s/TITLE/Wikimedia Traffic Analysis Report - Google requests/ ; $html =~ s/HEADER/Wikimedia Traffic Analysis Report - Google requests/ ; $html =~ s/ALSO/ See also: <b>LINKS<\/b>/ ; - $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/ $link_opsys \/ $link_devices \/ $link_browsers \/ $dummy_google \/ $link_countries \/ $link_ua_trends/ ; + $html =~ s/LINKS/$link_requests $link_origins \/ $link_methods \/ $link_scripts \/ $link_user_agents \/ $link_skins \/ $link_crawlers \/ $link_opsys \/ $link_devices \/ $link_browsers \/ $link_new_mobile_pageviews \/ $dummy_google \/ $link_countries \/ $link_ua_trends/ ; $html =~ s/X1000/⇒ <font color=#008000><b>all counts x 1000<\/b><\/font>.<br>/ ; $html .= "<table border=1 width=500 wrap>\n" ; diff --git a/squids/reports_basic/blank.sh b/squids/reports_basic/blank.sh new file mode 100644 index 0000000..92f72ed --- /dev/null +++ b/squids/reports_basic/blank.sh @@ -0,0 +1 @@ +blank -- To view, visit https://gerrit.wikimedia.org/r/59864 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ica902912862af002278a9b7b95803415806a550b Gerrit-PatchSet: 7 Gerrit-Project: analytics/wikistats Gerrit-Branch: master Gerrit-Owner: Stefan.petrea <[email protected]> Gerrit-Reviewer: Erik Zachte <[email protected]> Gerrit-Reviewer: jenkins-bot _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
