[MediaWiki-commits] [Gerrit] Lua func fixes - change (mediawiki...ZeroPortal)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/177133
Change subject: Lua func fixes
..
Lua func fixes
* isSiteAdmin() - proper handling of non-admin users
* isSiteAdmin() - return proper array-wrapped results
* setRawResult() - handle empty string as a valid value,
allow null to turn rawResult mode off
Change-Id: I108576635e08f3d278bd810fb44bd1727d14f52d
---
M includes/LuaLibrary.php
1 file changed, 36 insertions(+), 17 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroPortal
refs/changes/33/177133/1
diff --git a/includes/LuaLibrary.php b/includes/LuaLibrary.php
index 0746080..c666888 100644
--- a/includes/LuaLibrary.php
+++ b/includes/LuaLibrary.php
@@ -70,7 +70,7 @@
$resp-header( 'Cache-Control: ' . $cacheMode . ',
must-revalidate, max-age=0' );
}
- if ( self::$result ) {
+ if ( self::$result !== null ) {
$out-disable();
$resp-header( 'Content-Type: application/json;
charset=UTF-8' );
echo self::$result;
@@ -128,7 +128,7 @@
*/
public function getAllowedAccountIds() {
$configs = array();
- list( $name ) = $this-getUsername();
+ $name = $this-getUsernameInt();
if ( $name ) {
ApiZeroPortal::iterateAllConfigs( false,
function ( ZeroConfig $content, $title ) use (
$name, $configs ) {
@@ -147,12 +147,7 @@
* @throws Scribunto_LuaError
*/
public function getUsername() {
- $user = self::getContext()-getUser();
- if ( $user-isAnon() ) {
- return array( false );
- } else {
- return array( $user-getName() );
- }
+ return array( $this-getUsernameInt() );
}
/**
@@ -184,16 +179,17 @@
// First validate parameter, than analyze
$this-checkType( self::luaNamespace . __FUNCTION__, 1, $xcs,
'string' );
$title = new TitleValue( NS_ZERO, $xcs );
- if ( $this-isSiteAdmin() ) {
- return true;
+
+ if ( $this-isSiteAdminInt() ) {
+ return array( true );
}
- list( $name ) = $this-getUsername();
+ $name = $this-getUsernameInt();
if ( !$name ) {
- return false;
+ return array( false );
}
/** @var ZeroConfig $content */
$content = JCSingleton::getContent( $title );
- return in_array( $name, $content-admins() );
+ return array( in_array( $name, $content-admins() ) );
}
/**
@@ -202,7 +198,7 @@
* @throws Scribunto_LuaError
*/
public function isSiteAdmin() {
- return array( self::getContext()-getUser()-isAllowed(
'zero-edit' ) );
+ return array( $this-isSiteAdminInt() );
}
/**
@@ -245,12 +241,12 @@
}
/**
-* Set raw result of the special page
-* @param mixed $result
+* Set raw result of the special page, or null to disable raw mode. An
empty string is a valid result.
+* @param string|null $result
*/
public function setRawResult( $result = null ) {
self::getContext(); // ensure special page is enabled
- $this-checkType( self::luaNamespace . __FUNCTION__, 1,
$result, 'string' );
+ $this-checkTypeOptional( self::luaNamespace . __FUNCTION__, 1,
$result, 'string', null );
self::$result = $result;
}
@@ -263,4 +259,27 @@
return array( self::getContext()-getRequest()-wasPosted() );
}
+ /**
+* Return true if this is an admin user (anyone belonging to the
zero-edit group)
+* @return bool
+* @throws Scribunto_LuaError
+*/
+ private function isSiteAdminInt() {
+ return self::getContext()-getUser()-isAllowed( 'zero-edit' );
+ }
+
+ /**
+* Return name of the logged in user, or false if anonymous
+* @return string|false
+* @throws Scribunto_LuaError
+*/
+ public function getUsernameInt() {
+ $user = self::getContext()-getUser();
+ if ( $user-isAnon() ) {
+ return false;
+ } else {
+ return $user-getName();
+ }
+ }
+
}
--
To view, visit https://gerrit.wikimedia.org/r/177133
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I108576635e08f3d278bd810fb44bd1727d14f52d
[MediaWiki-commits] [Gerrit] Updated ZeroPortal to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/177140 Change subject: Updated ZeroPortal to master .. Updated ZeroPortal to master Change-Id: I7d0d7d4886141f1a31322337a50761418c07ec42 --- M extensions/ZeroPortal 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/40/177140/1 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 193e2cb..2fb5c62 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 193e2cb7e078b2d64bf0c5a6a666a57289cdbf79 +Subproject commit 2fb5c62c90860a0d550eedea436cc06e9874e92a -- To view, visit https://gerrit.wikimedia.org/r/177140 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I7d0d7d4886141f1a31322337a50761418c07ec42 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf10 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated ZeroBanner Portal to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/176719 Change subject: Updated ZeroBanner Portal to master .. Updated ZeroBanner Portal to master Change-Id: I29644d348b6e12ce3c83a17aabb15716264e9a64 --- M extensions/ZeroBanner M extensions/ZeroPortal 2 files changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/19/176719/1 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index 92253eb..04c1388 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit 92253eb6b7d5696bb1a4fb5a676d47ac0205b51d +Subproject commit 04c1388b9d83baa697dcad54e16d78935fb6f974 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 5740c76..193e2cb 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 5740c7631201d833e75afd43bc1e9aed8ef5158d +Subproject commit 193e2cb7e078b2d64bf0c5a6a666a57289cdbf79 -- To view, visit https://gerrit.wikimedia.org/r/176719 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I29644d348b6e12ce3c83a17aabb15716264e9a64 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf10 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated ZeroBanner Portal to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/176720 Change subject: Updated ZeroBanner Portal to master .. Updated ZeroBanner Portal to master Change-Id: I095027e0d9d3c0af9c2019111be918e062bde578 --- M extensions/ZeroBanner M extensions/ZeroPortal 2 files changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/20/176720/1 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index fdb0dba..04c1388 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit fdb0dba8aec41caeefe857236b20175eaece5f7a +Subproject commit 04c1388b9d83baa697dcad54e16d78935fb6f974 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index cf122c2..193e2cb 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit cf122c2b659e081f8cc0795dcca210a9117787f6 +Subproject commit 193e2cb7e078b2d64bf0c5a6a666a57289cdbf79 -- To view, visit https://gerrit.wikimedia.org/r/176720 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I095027e0d9d3c0af9c2019111be918e062bde578 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf9 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fix JsonEncode/Decode, error detection - change (mediawiki...ZeroPortal)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/176023
Change subject: Fix JsonEncode/Decode, error detection
..
Fix JsonEncode/Decode, error detection
* jsonDecode jsonEncode - recodes 1-based to 0-based
* getRawParameter() - better parameter error detection
Change-Id: I1d7376358fa70805fc9dbb02f535eeb7ef4ec5ed
---
M includes/LuaLibrary.php
1 file changed, 42 insertions(+), 9 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroPortal
refs/changes/23/176023/1
diff --git a/includes/LuaLibrary.php b/includes/LuaLibrary.php
index 116b312..0746080 100644
--- a/includes/LuaLibrary.php
+++ b/includes/LuaLibrary.php
@@ -80,6 +80,26 @@
}
}
+ private static function recurseValue( array $value, $isEncoding ) {
+ $isAssoc = false;
+ foreach ( $value as $k = $val ) {
+ if ( is_array( $val ) ) {
+ $val = self::recurseValue( $val, $isEncoding );
+ }
+ $isIntKey = is_int( $k ) || is_string( $k ) strval(
intval( $k ) ) === $k;
+ if ( !$isIntKey ) {
+ $isAssoc = true;
+ }
+ }
+ if ( $isAssoc ) {
+ return $value;
+ } elseif ( $isEncoding ) {
+ return array_values( $value );
+ } else {
+ return array_combine( range( 1, count( $value ) ),
$value );
+ }
+ }
+
public function register() {
$title = $this-getTitle();
if ( $title $title-inNamespace( NS_SPECIAL )
$title-getText() === 'ZeroPortal' ) {
@@ -138,13 +158,16 @@
/**
* Get a value from the request (GET or POST). Do not echo this value,
as it gets sent from the client
* @param string $name
-* @param mixed $default
-* @return mixed[]
+* @param string|null $default
+* @return string[]|null[]
* @throws Scribunto_LuaError
*/
public function getRawParameter( $name = null, $default = null ) {
$funcName = self::luaNamespace . __FUNCTION__;
$this-checkType( $funcName, 1, $name, 'string' );
+ if ( $default !== null ) {
+ $this-checkType( $funcName, 2, $default, 'string' );
+ }
if ( substr( $name, 0, 2 ) !== 'zp' ) {
throw new Scribunto_LuaError( bad argument #1 to
'$funcName' (name must start with 'zp') );
}
@@ -186,24 +209,34 @@
* Decode value as a JSON string.
* NOTE: this function uses FormatJson::decode() for simplicity sake.
* TODO: remove this function once Scribunto gets its own json
encode/decode
-* @param mixed $value
+* @param string $value
+* @param bool $preserveKeys
+* @return \mixed[]
* @throws Scribunto_LuaError
-* @return mixed[]
*/
- public function jsonDecode( $value = null ) {
+ public function jsonDecode( $value = null, $preserveKeys = null ) {
$this-checkType( self::luaNamespace . __FUNCTION__, 1, $value,
'string' );
- return array( FormatJson::decode( $value, true ) );
+ $this-checkTypeOptional( self::luaNamespace . __FUNCTION__, 2,
$preserveKeys, 'boolean', false );
+ $val = FormatJson::decode( $value, true );
+ if ( !$preserveKeys is_array( $val ) ) {
+ $val = self::recurseValue( $val, false );
+ }
+ return array( $val );
}
/**
* Encode value as a JSON string.
* TODO: remove this function once Scribunto gets its own json
encode/decode
* @param mixed $value
-* @throws Scribunto_LuaError
+* @param bool $preserveKeys
* @return string
+* @throws Scribunto_LuaError
*/
- public function jsonEncode( $value = null ) {
- $this-checkTypeOptional( self::luaNamespace . __FUNCTION__, 2,
$escapeHtml, 'boolean', false );
+ public function jsonEncode( $value = null, $preserveKeys = null ) {
+ $this-checkTypeOptional( self::luaNamespace . __FUNCTION__, 2,
$preserveKeys, 'boolean', false );
+ if ( !$preserveKeys is_array( $value ) ) {
+ $value = self::recurseValue( $value, true );
+ }
$result = FormatJson::encode( $value, false, FormatJson::ALL_OK
);
if ( $result === false ) {
throw new Scribunto_LuaError( 'Unable to encode value'
);
--
To view, visit https://gerrit.wikimedia.org/r/176023
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
[MediaWiki-commits] [Gerrit] fix broken xml in .idea/codeStyleSettings.xml - change (mediawiki/vagrant)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/175617 Change subject: fix broken xml in .idea/codeStyleSettings.xml .. fix broken xml in .idea/codeStyleSettings.xml Change-Id: Ic3697732aae4270ab22ceb9f335e3e1a003a3c3f --- M .idea/codeStyleSettings.xml 1 file changed, 1 insertion(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant refs/changes/17/175617/1 diff --git a/.idea/codeStyleSettings.xml b/.idea/codeStyleSettings.xml index 60ae90c..54fd6a5 100644 --- a/.idea/codeStyleSettings.xml +++ b/.idea/codeStyleSettings.xml @@ -79,6 +79,7 @@ indentOptions option name=INDENT_SIZE value=2 / option name=CONTINUATION_INDENT_SIZE value=2 / + /indentOptions /codeStyleSettings /value /option -- To view, visit https://gerrit.wikimedia.org/r/175617 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ic3697732aae4270ab22ceb9f335e3e1a003a3c3f Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/vagrant Gerrit-Branch: master Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fix PHP Notice: Undefined offset: 1 bug - change (mediawiki...ZeroBanner)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/173744
Change subject: Fix PHP Notice: Undefined offset: 1 bug
..
Fix PHP Notice: Undefined offset: 1 bug
bug 73486
Change-Id: I25d13d1d20db8234f79e2cc7d9baa2bdde55fd73
---
M includes/PageRendering.php
1 file changed, 5 insertions(+), 4 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroBanner
refs/changes/44/173744/1
diff --git a/includes/PageRendering.php b/includes/PageRendering.php
index 3cb7456..d2e8ed3 100644
--- a/includes/PageRendering.php
+++ b/includes/PageRendering.php
@@ -584,16 +584,17 @@
if ( $this-config === false || ( $this-config
$this-config-contextMode() !== $mode ) ) {
wfProfileIn( __METHOD__ );
+ $ipset = '';
$xcs = $this-getConfigId();
// Unified is treated as if there is no config - we
don't know the actual ID
if ( $xcs !== null !$this-isUnifiedZero() ) {
- @list( $id, $ipset ) = explode( '|', $xcs, 2 );
- if ( $ipset === null ) {
- $ipset = '';
+ $xcsParts = explode( '|', $xcs, 2 );
+ $id = $xcsParts[0];
+ if ( count( $xcsParts ) 1 ) {
+ $ipset = $xcsParts[1];
}
} else {
$id = null;
- $ipset = '';
}
if ( $id !== null $this-config === false ) {
--
To view, visit https://gerrit.wikimedia.org/r/173744
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I25d13d1d20db8234f79e2cc7d9baa2bdde55fd73
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/ZeroBanner
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated ZeroPortal to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/173001 Change subject: Updated ZeroPortal to master .. Updated ZeroPortal to master Change-Id: Ib653999ee08399c99223cb1d9af20b57923f76bb --- M extensions/ZeroPortal 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/01/173001/1 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 4b889c7..4cac813 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 4b889c77be5c1bd26c9da45a373111acc1ba55ee +Subproject commit 4cac813903f2666d73cad1bc6e8233b21644774e -- To view, visit https://gerrit.wikimedia.org/r/173001 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ib653999ee08399c99223cb1d9af20b57923f76bb Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf8 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated ZeroPortal to master - change (mediawiki/core)
Yurik has submitted this change and it was merged. Change subject: Updated ZeroPortal to master .. Updated ZeroPortal to master Change-Id: Ib653999ee08399c99223cb1d9af20b57923f76bb --- M extensions/ZeroPortal 1 file changed, 0 insertions(+), 0 deletions(-) Approvals: Yurik: Verified; Looks good to me, approved diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 4b889c7..4cac813 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 4b889c77be5c1bd26c9da45a373111acc1ba55ee +Subproject commit 4cac813903f2666d73cad1bc6e8233b21644774e -- To view, visit https://gerrit.wikimedia.org/r/173001 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ib653999ee08399c99223cb1d9af20b57923f76bb Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf8 Gerrit-Owner: Yurik yu...@wikimedia.org Gerrit-Reviewer: Yurik yu...@wikimedia.org Gerrit-Reviewer: jenkins-bot ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated weblogs2 to create single pages - change (analytics/zero-sms)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/172661 Change subject: Updated weblogs2 to create single pages .. Updated weblogs2 to create single pages Change-Id: Ic2789a8478b01ee3117385f5e44eaf383c0db9f3 --- M scripts/run-hivezero.sh M scripts/weblogs2.py 2 files changed, 48 insertions(+), 80 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms refs/changes/61/172661/1 diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh index 2c8f95c..7688adc 100755 --- a/scripts/run-hivezero.sh +++ b/scripts/run-hivezero.sh @@ -31,7 +31,7 @@ if [ $( date -d $date +%F 21 | grep invalid ) = ] ; then if [[ $table == 'wmf_raw.webrequest' ]]; then - path=/mnt/hdfs/wmf/data/raw/webrequest/webrequest_upload/hourly/$year/$month/$day/23 + path=/mnt/hdfs/wmf/data/raw/webrequest/webrequest_upload/hourly/$year/$(printf %02d $month)/$(printf %02d $day)/23 else path=/mnt/hdfs/user/hive/warehouse/yurik.db/$table/year=$year/month=$month/day=$day fi @@ -45,12 +45,14 @@ continue fi - if [[ $6 -eq overwrite ]]; then - hive -e use yurik; ALTER TABLE zero_webstats DROP IF EXISTS PARTITION(date = '$date'); - else - path=/mnt/hdfs/user/hive/warehouse/yurik.db/zero_webstats/date=$date - echo * Checking if '$path' exists - if [ -d $path ]; then + path=/mnt/hdfs/user/hive/warehouse/yurik.db/zero_webstats/date=$date + echo * Checking if '$path' exists + if [ -d $path ]; then + if [ $6 == overwrite ]; then + echo * Droping partition '$date' + hive -e use yurik; ALTER TABLE zero_webstats DROP IF EXISTS PARTITION(date = '$date'); + else + echo * Skipping '$date' continue fi fi diff --git a/scripts/weblogs2.py b/scripts/weblogs2.py index 1e79027..1e0f727 100644 --- a/scripts/weblogs2.py +++ b/scripts/weblogs2.py @@ -4,7 +4,7 @@ import collections from pandas import read_table, pivot_table -from pandas.core.frame import DataFrame, Series +from pandas.core.frame import DataFrame import numpy as np from logprocessor import * @@ -58,13 +58,13 @@ ignoreViaBefore = datetime(2014, 3, 22) configs = self.downloadConfigs() stats = collections.defaultdict(int) -for dateDir in os.listdir(self.pathCache): +for dateDir in os.listdir(self.pathLogs): m = self.dateDirRe.match(dateDir) if not m: continue dateStr = m.group(1) dt = datetime.strptime(dateStr, '%Y-%m-%d') -datePath = os.path.join(self.pathCache, dateDir) +datePath = os.path.join(self.pathLogs, dateDir) for f in os.listdir(datePath): if not self.fileRe.match(f): continue @@ -136,90 +136,56 @@ xcsList = [xcs for xcs in allData.xcs.unique() if xcs != 'ERROR' and xcs[0:4] != 'TEST'] # filter type==DATA and site==wikipedia -allData = allData[(allData['xcs'].isin(xcsList)) (allData['site'] == 'wikipedia')] -# filter out last date -lastDate = allData.date.max() -df = allData[allData.date lastDate] +df = allData[(allData['xcs'].isin(xcsList)) (allData['site'] == 'wikipedia')] -allowedSubdomains = ['m', 'zero'] -limnCompat = df[(df.ison == 'y') (df.iszero == 'y') (df.subdomain.isin(allowedSubdomains))] s = StringIO.StringIO() -pivot_table(limnCompat, 'count', ['date', 'xcs', 'subdomain'], aggfunc=np.sum).to_csv(s, header=True) -result = s.getvalue() +allowedSubdomains = ['m', 'zero'] +dailySubdomains = df[(df.ison == 'y') (df.iszero == 'y') (df.subdomain.isin(allowedSubdomains))] +pivot_table(dailySubdomains, 'count', ['date', 'xcs', 'subdomain'], aggfunc=np.sum).to_csv(s, header=False) wiki( 'edit', title='RawData:DailySubdomains', summary='refreshing data', -text=result, +text='date,xcs,subdomain,count\n' + s.getvalue(), token=wiki.token() ) -# allEnabled = df[(df.ison == 'y') (df.iszero == 'y')] -# s = StringIO.StringIO() -# pivot_table(allEnabled, 'count', ['date', 'xcs'], aggfunc=np.sum).to_csv(s, header=True) -# result = s.getvalue() -# -# wiki( -# 'edit', -# title='RawData:AllEnabled', -# summary='refreshing data',
[MediaWiki-commits] [Gerrit] Updated weblogs2 to create single pages - change (analytics/zero-sms)
Yurik has submitted this change and it was merged. Change subject: Updated weblogs2 to create single pages .. Updated weblogs2 to create single pages Change-Id: Ic2789a8478b01ee3117385f5e44eaf383c0db9f3 --- M scripts/run-hivezero.sh M scripts/weblogs2.py 2 files changed, 48 insertions(+), 80 deletions(-) Approvals: Yurik: Verified; Looks good to me, approved diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh index 2c8f95c..7688adc 100755 --- a/scripts/run-hivezero.sh +++ b/scripts/run-hivezero.sh @@ -31,7 +31,7 @@ if [ $( date -d $date +%F 21 | grep invalid ) = ] ; then if [[ $table == 'wmf_raw.webrequest' ]]; then - path=/mnt/hdfs/wmf/data/raw/webrequest/webrequest_upload/hourly/$year/$month/$day/23 + path=/mnt/hdfs/wmf/data/raw/webrequest/webrequest_upload/hourly/$year/$(printf %02d $month)/$(printf %02d $day)/23 else path=/mnt/hdfs/user/hive/warehouse/yurik.db/$table/year=$year/month=$month/day=$day fi @@ -45,12 +45,14 @@ continue fi - if [[ $6 -eq overwrite ]]; then - hive -e use yurik; ALTER TABLE zero_webstats DROP IF EXISTS PARTITION(date = '$date'); - else - path=/mnt/hdfs/user/hive/warehouse/yurik.db/zero_webstats/date=$date - echo * Checking if '$path' exists - if [ -d $path ]; then + path=/mnt/hdfs/user/hive/warehouse/yurik.db/zero_webstats/date=$date + echo * Checking if '$path' exists + if [ -d $path ]; then + if [ $6 == overwrite ]; then + echo * Droping partition '$date' + hive -e use yurik; ALTER TABLE zero_webstats DROP IF EXISTS PARTITION(date = '$date'); + else + echo * Skipping '$date' continue fi fi diff --git a/scripts/weblogs2.py b/scripts/weblogs2.py index 1e79027..1e0f727 100644 --- a/scripts/weblogs2.py +++ b/scripts/weblogs2.py @@ -4,7 +4,7 @@ import collections from pandas import read_table, pivot_table -from pandas.core.frame import DataFrame, Series +from pandas.core.frame import DataFrame import numpy as np from logprocessor import * @@ -58,13 +58,13 @@ ignoreViaBefore = datetime(2014, 3, 22) configs = self.downloadConfigs() stats = collections.defaultdict(int) -for dateDir in os.listdir(self.pathCache): +for dateDir in os.listdir(self.pathLogs): m = self.dateDirRe.match(dateDir) if not m: continue dateStr = m.group(1) dt = datetime.strptime(dateStr, '%Y-%m-%d') -datePath = os.path.join(self.pathCache, dateDir) +datePath = os.path.join(self.pathLogs, dateDir) for f in os.listdir(datePath): if not self.fileRe.match(f): continue @@ -136,90 +136,56 @@ xcsList = [xcs for xcs in allData.xcs.unique() if xcs != 'ERROR' and xcs[0:4] != 'TEST'] # filter type==DATA and site==wikipedia -allData = allData[(allData['xcs'].isin(xcsList)) (allData['site'] == 'wikipedia')] -# filter out last date -lastDate = allData.date.max() -df = allData[allData.date lastDate] +df = allData[(allData['xcs'].isin(xcsList)) (allData['site'] == 'wikipedia')] -allowedSubdomains = ['m', 'zero'] -limnCompat = df[(df.ison == 'y') (df.iszero == 'y') (df.subdomain.isin(allowedSubdomains))] s = StringIO.StringIO() -pivot_table(limnCompat, 'count', ['date', 'xcs', 'subdomain'], aggfunc=np.sum).to_csv(s, header=True) -result = s.getvalue() +allowedSubdomains = ['m', 'zero'] +dailySubdomains = df[(df.ison == 'y') (df.iszero == 'y') (df.subdomain.isin(allowedSubdomains))] +pivot_table(dailySubdomains, 'count', ['date', 'xcs', 'subdomain'], aggfunc=np.sum).to_csv(s, header=False) wiki( 'edit', title='RawData:DailySubdomains', summary='refreshing data', -text=result, +text='date,xcs,subdomain,count\n' + s.getvalue(), token=wiki.token() ) -# allEnabled = df[(df.ison == 'y') (df.iszero == 'y')] -# s = StringIO.StringIO() -# pivot_table(allEnabled, 'count', ['date', 'xcs'], aggfunc=np.sum).to_csv(s, header=True) -# result = s.getvalue() -# -# wiki( -# 'edit', -# title='RawData:AllEnabled', -# summary='refreshing data', -# text=result, -# token=wiki.token() -
[MediaWiki-commits] [Gerrit] Updated scripts - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/171810
Change subject: Updated scripts
..
Updated scripts
Change-Id: I101466924217d19b4def63a4a4ae8b5e3c915a7a
---
M scripts/api.py
M scripts/log2dfs.py
M scripts/logprocessor.py
M scripts/run-hivezero.sh
M scripts/smslogs.py
M scripts/weblogs.py
M scripts/weblogs2.py
M scripts/zero-counts.hql
8 files changed, 141 insertions(+), 134 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/10/171810/1
diff --git a/scripts/api.py b/scripts/api.py
index 4037aeb..ed0bf6c 100644
--- a/scripts/api.py
+++ b/scripts/api.py
@@ -86,6 +86,7 @@
def __init__(self, url, headers=None, session=None, log=None):
+self._loginOnDemand = False
self.session = session if session else requests.session()
self.log = log if log else ConsoleLog()
self.url = url
@@ -143,6 +144,9 @@
else:
request_kw['params'] = kwargs
+if self._loginOnDemand and action != 'login':
+self.login(self._loginOnDemand[0], self._loginOnDemand[1])
+
data = parseJson(self.request(method, forceSSL=forceSSL, **request_kw))
# Handle success and failure
@@ -152,13 +156,23 @@
self.log(2, data['warnings'])
return data
-def login(self, user, password):
+def login(self, user, password, onDemand=False):
+
+:param user:
+:param password:
+:param onDemand: if True, will postpone login until an actual API
request is made
+:return:
+
+if onDemand:
+self._loginOnDemand = (user, password)
+return
self.tokens = {}
res = self('login', lgname=user, lgpassword=password)['login']
if res['result'] == 'NeedToken':
res = self('login', lgname=user, lgpassword=password,
lgtoken=res['token'])['login']
if res['result'] != 'Success':
raise ApiError('Login failed', res)
+self._loginOnDemand = False
def query(self, **kwargs):
diff --git a/scripts/log2dfs.py b/scripts/log2dfs.py
index 30aa8d5..9f888ec 100644
--- a/scripts/log2dfs.py
+++ b/scripts/log2dfs.py
@@ -221,7 +221,7 @@
safePrint(u'URL parsing failed: %s\n%s' % (uri,
line))
continue
if m.group(1).lower() == u'https' and u'https=' not in
x_analytics:
-x_analytics += u'https=1'
+x_analytics += u';https=1'
uri_host = m.group(2)
if uri_host.endswith(':80'):
uri_host = uri_host[:-3]
diff --git a/scripts/logprocessor.py b/scripts/logprocessor.py
index 0e425b1..f74f128 100644
--- a/scripts/logprocessor.py
+++ b/scripts/logprocessor.py
@@ -188,7 +188,7 @@
self._wiki = api.wikimedia('zero', 'wikimedia', 'https')
if self.proxy:
self._wiki.session.proxies = {'http': 'http://%s:%d' %
(self.proxy, self.proxyPort)}
-self._wiki.login(self.settings.apiUsername,
self.settings.apiPassword)
+self._wiki.login(self.settings.apiUsername,
self.settings.apiPassword, onDemand=True)
return self._wiki
# noinspection PyMethodMayBeStatic
@@ -217,12 +217,11 @@
def __init__(self, settingsFile, pathSuffix):
super(LogProcessor, self).__init__(settingsFile, pathSuffix)
-if not self.settings.pathLogs or not self.settings.pathCache or not
self.settings.pathGraphs:
+if not self.settings.pathLogs or not self.settings.pathCache:
raise ValueError('One of the paths is not set, check %s' %
settingsFile)
self.pathLogs = self.normalizePath(self.settings.pathLogs)
self.pathCache = self.normalizePath(self.settings.pathCache)
-self.pathGraphs = self.normalizePath(self.settings.pathGraphs)
def defaultSettings(self, suffix):
s = super(LogProcessor, self).defaultSettings(suffix)
@@ -231,5 +230,4 @@
suffix = os.sep + suffix if suffix else ''
s.pathLogs = 'logs' + suffix
s.pathCache = 'cache' + suffix
-s.pathGraphs = 'graphs' + suffix
return s
diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh
index d1f799b..2c8f95c 100755
--- a/scripts/run-hivezero.sh
+++ b/scripts/run-hivezero.sh
@@ -1,16 +1,63 @@
#!/bin/bash
-#$1 $2 $3 $4 $5 %6 $7
-# ./run-clone.sh wmf_raw.webrequest 515-05 2014 10 11 0 23
+# $1 $2 $3 $4 %5 $6
+# ./run-hivezero.sh wmf_raw.webrequest 2014 10 1 31
+# ./run-hivezero.sh webreq_archive 2014 10 1 31 overwrite
-if [[ -z $7 ]]; then
- last=$6
+set -e
+
+if [[ -z $5 ]]; then
+ last=$4
else
- last=$7
+ last=$5
fi
-for ((hour = $6; hour = $last; hour++)); do
-
[MediaWiki-commits] [Gerrit] Updated scripts - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: Updated scripts
..
Updated scripts
Change-Id: I101466924217d19b4def63a4a4ae8b5e3c915a7a
---
M scripts/api.py
M scripts/log2dfs.py
M scripts/logprocessor.py
M scripts/run-hivezero.sh
M scripts/smslogs.py
M scripts/weblogs.py
M scripts/weblogs2.py
M scripts/zero-counts.hql
8 files changed, 141 insertions(+), 134 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/api.py b/scripts/api.py
index 4037aeb..ed0bf6c 100644
--- a/scripts/api.py
+++ b/scripts/api.py
@@ -86,6 +86,7 @@
def __init__(self, url, headers=None, session=None, log=None):
+self._loginOnDemand = False
self.session = session if session else requests.session()
self.log = log if log else ConsoleLog()
self.url = url
@@ -143,6 +144,9 @@
else:
request_kw['params'] = kwargs
+if self._loginOnDemand and action != 'login':
+self.login(self._loginOnDemand[0], self._loginOnDemand[1])
+
data = parseJson(self.request(method, forceSSL=forceSSL, **request_kw))
# Handle success and failure
@@ -152,13 +156,23 @@
self.log(2, data['warnings'])
return data
-def login(self, user, password):
+def login(self, user, password, onDemand=False):
+
+:param user:
+:param password:
+:param onDemand: if True, will postpone login until an actual API
request is made
+:return:
+
+if onDemand:
+self._loginOnDemand = (user, password)
+return
self.tokens = {}
res = self('login', lgname=user, lgpassword=password)['login']
if res['result'] == 'NeedToken':
res = self('login', lgname=user, lgpassword=password,
lgtoken=res['token'])['login']
if res['result'] != 'Success':
raise ApiError('Login failed', res)
+self._loginOnDemand = False
def query(self, **kwargs):
diff --git a/scripts/log2dfs.py b/scripts/log2dfs.py
index 30aa8d5..9f888ec 100644
--- a/scripts/log2dfs.py
+++ b/scripts/log2dfs.py
@@ -221,7 +221,7 @@
safePrint(u'URL parsing failed: %s\n%s' % (uri,
line))
continue
if m.group(1).lower() == u'https' and u'https=' not in
x_analytics:
-x_analytics += u'https=1'
+x_analytics += u';https=1'
uri_host = m.group(2)
if uri_host.endswith(':80'):
uri_host = uri_host[:-3]
diff --git a/scripts/logprocessor.py b/scripts/logprocessor.py
index 0e425b1..f74f128 100644
--- a/scripts/logprocessor.py
+++ b/scripts/logprocessor.py
@@ -188,7 +188,7 @@
self._wiki = api.wikimedia('zero', 'wikimedia', 'https')
if self.proxy:
self._wiki.session.proxies = {'http': 'http://%s:%d' %
(self.proxy, self.proxyPort)}
-self._wiki.login(self.settings.apiUsername,
self.settings.apiPassword)
+self._wiki.login(self.settings.apiUsername,
self.settings.apiPassword, onDemand=True)
return self._wiki
# noinspection PyMethodMayBeStatic
@@ -217,12 +217,11 @@
def __init__(self, settingsFile, pathSuffix):
super(LogProcessor, self).__init__(settingsFile, pathSuffix)
-if not self.settings.pathLogs or not self.settings.pathCache or not
self.settings.pathGraphs:
+if not self.settings.pathLogs or not self.settings.pathCache:
raise ValueError('One of the paths is not set, check %s' %
settingsFile)
self.pathLogs = self.normalizePath(self.settings.pathLogs)
self.pathCache = self.normalizePath(self.settings.pathCache)
-self.pathGraphs = self.normalizePath(self.settings.pathGraphs)
def defaultSettings(self, suffix):
s = super(LogProcessor, self).defaultSettings(suffix)
@@ -231,5 +230,4 @@
suffix = os.sep + suffix if suffix else ''
s.pathLogs = 'logs' + suffix
s.pathCache = 'cache' + suffix
-s.pathGraphs = 'graphs' + suffix
return s
diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh
index d1f799b..2c8f95c 100755
--- a/scripts/run-hivezero.sh
+++ b/scripts/run-hivezero.sh
@@ -1,16 +1,63 @@
#!/bin/bash
-#$1 $2 $3 $4 $5 %6 $7
-# ./run-clone.sh wmf_raw.webrequest 515-05 2014 10 11 0 23
+# $1 $2 $3 $4 %5 $6
+# ./run-hivezero.sh wmf_raw.webrequest 2014 10 1 31
+# ./run-hivezero.sh webreq_archive 2014 10 1 31 overwrite
-if [[ -z $7 ]]; then
- last=$6
+set -e
+
+if [[ -z $5 ]]; then
+ last=$4
else
- last=$7
+ last=$5
fi
-for ((hour = $6; hour = $last; hour++)); do
- printf -v t tmp_%04d_%02d_%02d_%02d $3 $4 $5 $hour
-
[MediaWiki-commits] [Gerrit] Re-implemented missing Zero Portal Lua function - change (mediawiki...ZeroPortal)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/171472
Change subject: Re-implemented missing Zero Portal Lua function
..
Re-implemented missing Zero Portal Lua function
* getAccountsForUser()
* getUsername()
* getVal( name, default )
* isAdmin()
* jsonDecode( value )
* jsonEncode( value, escapeHtml )
* wasPosted()
Change-Id: I8ad0f6253d34ba93b3f6aa3bcbfd42129680be7e
---
M includes/LuaLibrary.lua
M includes/LuaLibrary.php
2 files changed, 135 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroPortal
refs/changes/72/171472/1
diff --git a/includes/LuaLibrary.lua b/includes/LuaLibrary.lua
index 04be097..c70eced 100644
--- a/includes/LuaLibrary.lua
+++ b/includes/LuaLibrary.lua
@@ -17,8 +17,37 @@
-- Each of the following functions should be documented in the corresponding
LuaLibrary.php function
+function p.getAccountsForUser()
+return php.getAccountsForUser()
+end
+
+function p.getUsername()
+return php.getUsername()
+end
+
+function p.getVal( name, default )
+return php.getVal( name, default )
+end
+
+function p.isAdmin()
+return php.isAdmin()
+end
+
+function p.jsonDecode( value )
+return php.jsonDecode( value )
+end
+
+function p.jsonEncode( value, escapeHtml )
+return php.jsonEncode( value, escapeHtml )
+end
+
function p.setRawResult( result )
return php.setRawResult( result )
end
+function p.wasPosted()
+return php.wasPosted()
+end
+
+
return p
diff --git a/includes/LuaLibrary.php b/includes/LuaLibrary.php
index d60ab96..0ece57b 100644
--- a/includes/LuaLibrary.php
+++ b/includes/LuaLibrary.php
@@ -2,9 +2,11 @@
namespace ZeroPortal;
+use FormatJson;
use Scribunto_LuaError;
use Scribunto_LuaLibraryBase;
use IContextSource;
+use ZeroBanner\ZeroConfig;
class LuaLibrary extends Scribunto_LuaLibraryBase {
@@ -81,13 +83,108 @@
if ( $title $title-inNamespace( NS_SPECIAL )
$title-getText() === 'ZeroPortal' ) {
$functions = array();
foreach ( array(
+ 'getAccountsForUser',
+ 'getUsername',
+ 'getVal',
+ 'isAdmin',
+ 'jsonDecode',
+ 'jsonEncode',
'setRawResult',
+ 'wasPosted',
) as $f ) {
$functions[$f] = array( $this, $f );
}
$moduleFileName = __DIR__ . DIRECTORY_SEPARATOR .
'LuaLibrary.lua';
$this-getEngine()-registerInterface( $moduleFileName,
$functions, array() );
}
+ }
+
+ /**
+* Returns a list of Zero config titles that this user has been
assigned to as an admin
+* @return string[]|false[]
+*/
+ public function getAccountsForUser() {
+ $configs = array();
+ list( $name ) = $this-getUsername();
+ if ( $name ) {
+ ApiZeroPortal::iterateAllConfigs( false,
+ function ( ZeroConfig $content, $title ) use (
$name, $configs ) {
+ if ( $name === true || in_array( $name,
$content-admins() ) ) {
+ $configs[] = $title;
+ }
+ return false;
+ } );
+ }
+ return array( $configs );
+ }
+
+ /**
+* Return name of the logged in user, or false if anonymous
+* @return string[]|bool[]
+* @throws Scribunto_LuaError
+*/
+ public function getUsername() {
+ $user = self::getContext()-getUser();
+ if ( $user-isAnon() ) {
+ return array( false );
+ } else {
+ return array( $user-getName() );
+ }
+ }
+
+ /**
+* Get a value from the request (GET or POST).
+* @param string $name
+* @param mixed $default
+* @return mixed[]
+* @throws Scribunto_LuaError
+*/
+ public function getVal( $name = null, $default = null ) {
+ $funcName = self::luaNamespace . __FUNCTION__;
+ $this-checkType( $funcName, 1, $name, 'string' );
+ if ( substr( $name, 0, 2 ) !== 'zp' ) {
+ throw new Scribunto_LuaError( bad argument #1 to
'$funcName' (name must start with 'zp') );
+ }
+ return array( self::getContext()-getRequest()-getVal( $name,
$default ) );
+ }
+
+
[MediaWiki-commits] [Gerrit] updated scripts (misc changes) - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/169750
Change subject: updated scripts (misc changes)
..
updated scripts (misc changes)
Change-Id: Ib4a75794769c4aaaf155718cb9f1151a1ef4266b
---
M scripts/log2dfs.py
M scripts/run-hivezero.sh
M scripts/zero-counts.hql
3 files changed, 49 insertions(+), 32 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/50/169750/1
diff --git a/scripts/log2dfs.py b/scripts/log2dfs.py
index a4d7b51..30aa8d5 100644
--- a/scripts/log2dfs.py
+++ b/scripts/log2dfs.py
@@ -61,7 +61,7 @@
'FAKE_CACHE_STATUS': '',
'TCP_HIT': 'hit',
'TCP_IMS_HIT': 'hit',
-'TCP_DENIED': '',
+'TCP_DENIED': 'denied',
'TCP_REFRESH_MISS': 'miss',
'TCP_NEGATIVE_HIT': 'hit',
}
@@ -76,6 +76,7 @@
self.logFileRe = re.compile(unicode(filePattern), re.IGNORECASE)
self.dateRe = re.compile(r'(201\d-\d\d-\d\dT\d\d):\d\d:\d\d(\.\d+)?')
self.urlRe = re.compile(r'^(https?)://([^/]+)([^?#]*)(.*)',
re.IGNORECASE)
+self.xcsRe = re.compile(r'^[0-9]+-[0-9]+$', re.IGNORECASE)
def processLogFiles(self):
@@ -89,8 +90,7 @@
if statFile.endswith('.gz'):
statFile = statFile[:-3]
-# if not os.path.exists(statFile):
-if True:
+if not os.path.exists(statFile):
self.processLogFile(logFile, statFile)
def processLogFile(self, logFile, statFile):
@@ -133,26 +133,26 @@
else:
streamData = io.open(logFile, 'r', encoding='utf8',
errors='ignore')
-with io.open(statFile, 'w', encoding='utf8') as out:
+tmpFile = statFile + '.tmp'
+with io.open(tmpFile, 'w', encoding='utf8') as out:
for line in streamData:
count += 1
if count % 100 == 0:
safePrint('%d lines processed' % count)
-strip = line.strip('\n\r')
+strip = line.strip('\n\r\x00')
if isTab:
l = strip.split('\t')
-if l[2].startswith('201'):
+if len(l) 2 and l[2].startswith('201'):
while len(l) 16:
l[13] += ' ' + l[14]
del l[11]
else:
l = strip.split(' ')
# fix text/html; charset=UTF-8 into one field
-while len(l) = 10 and l[10].endswith(';') and l[11] !=
'-' and not l[11].startswith('http'):
+while len(l) 11 and l[10].endswith(';') and l[11] != '-'
and not l[11].startswith('http'):
l[10] += ' ' + l[11]
del l[11]
-l[13] = unquote(l[13])
if len(l) == 14:
l.append(u'')
l.append(u'')
@@ -162,12 +162,14 @@
safePrint(u'Wrong parts count - %d parts\n%s' %
(partsCount, line))
continue
-l = ['' if v == '-' else v for v in l]
+l = ['' if v == '-' else v.replace('\t', ' ') for v in l]
(hostname, sequence, dt, time_firstbyte, ip, status,
response_size, http_method, uri, unknown1,
content_type, referer, x_forwarded_for, user_agent,
accept_language, x_analytics) = l
# status - cache_status, http_status
# uri - uri_host, uri_path, uri_query
# new: webrequest_source, year, month, day, hour
+
+user_agent = unquote(user_agent).replace('\t', ' ')
m = self.dateRe.match(dt)
if not m:
@@ -180,6 +182,9 @@
month = unicode(d.month)
day = unicode(d.day)
hour = unicode(d.hour)
+
+if self.xcsRe.match(x_analytics):
+x_analytics = 'zero=' + x_analytics
if 'zero=' not in x_analytics:
if defaultXcs:
@@ -207,19 +212,23 @@
continue
cache_status = httpStatuses[cache_status]
-m = self.urlRe.match(uri)
-if not m:
-safePrint(u'URL parsing failed: %s\n%s' % (uri, line))
-continue
-if m.group(1).lower() == u'https' and u'https=' not in
x_analytics:
-x_analytics += u'https=1'
-uri_host = m.group(2)
-if uri_host.endswith(':80'):
-uri_host = uri_host[:-3]
-if uri_host.endswith('.'):
-uri_host = uri_host[:-1]
-uri_path = m.group(3)
-uri_query = m.group(4)
+if uri == 'NONE://' or (http_method == 'CONNECT' and uri ==
':0'):
+
[MediaWiki-commits] [Gerrit] updated scripts (misc changes) - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: updated scripts (misc changes)
..
updated scripts (misc changes)
Change-Id: Ib4a75794769c4aaaf155718cb9f1151a1ef4266b
---
M scripts/log2dfs.py
M scripts/run-hivezero.sh
M scripts/zero-counts.hql
3 files changed, 49 insertions(+), 32 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/log2dfs.py b/scripts/log2dfs.py
index a4d7b51..30aa8d5 100644
--- a/scripts/log2dfs.py
+++ b/scripts/log2dfs.py
@@ -61,7 +61,7 @@
'FAKE_CACHE_STATUS': '',
'TCP_HIT': 'hit',
'TCP_IMS_HIT': 'hit',
-'TCP_DENIED': '',
+'TCP_DENIED': 'denied',
'TCP_REFRESH_MISS': 'miss',
'TCP_NEGATIVE_HIT': 'hit',
}
@@ -76,6 +76,7 @@
self.logFileRe = re.compile(unicode(filePattern), re.IGNORECASE)
self.dateRe = re.compile(r'(201\d-\d\d-\d\dT\d\d):\d\d:\d\d(\.\d+)?')
self.urlRe = re.compile(r'^(https?)://([^/]+)([^?#]*)(.*)',
re.IGNORECASE)
+self.xcsRe = re.compile(r'^[0-9]+-[0-9]+$', re.IGNORECASE)
def processLogFiles(self):
@@ -89,8 +90,7 @@
if statFile.endswith('.gz'):
statFile = statFile[:-3]
-# if not os.path.exists(statFile):
-if True:
+if not os.path.exists(statFile):
self.processLogFile(logFile, statFile)
def processLogFile(self, logFile, statFile):
@@ -133,26 +133,26 @@
else:
streamData = io.open(logFile, 'r', encoding='utf8',
errors='ignore')
-with io.open(statFile, 'w', encoding='utf8') as out:
+tmpFile = statFile + '.tmp'
+with io.open(tmpFile, 'w', encoding='utf8') as out:
for line in streamData:
count += 1
if count % 100 == 0:
safePrint('%d lines processed' % count)
-strip = line.strip('\n\r')
+strip = line.strip('\n\r\x00')
if isTab:
l = strip.split('\t')
-if l[2].startswith('201'):
+if len(l) 2 and l[2].startswith('201'):
while len(l) 16:
l[13] += ' ' + l[14]
del l[11]
else:
l = strip.split(' ')
# fix text/html; charset=UTF-8 into one field
-while len(l) = 10 and l[10].endswith(';') and l[11] !=
'-' and not l[11].startswith('http'):
+while len(l) 11 and l[10].endswith(';') and l[11] != '-'
and not l[11].startswith('http'):
l[10] += ' ' + l[11]
del l[11]
-l[13] = unquote(l[13])
if len(l) == 14:
l.append(u'')
l.append(u'')
@@ -162,12 +162,14 @@
safePrint(u'Wrong parts count - %d parts\n%s' %
(partsCount, line))
continue
-l = ['' if v == '-' else v for v in l]
+l = ['' if v == '-' else v.replace('\t', ' ') for v in l]
(hostname, sequence, dt, time_firstbyte, ip, status,
response_size, http_method, uri, unknown1,
content_type, referer, x_forwarded_for, user_agent,
accept_language, x_analytics) = l
# status - cache_status, http_status
# uri - uri_host, uri_path, uri_query
# new: webrequest_source, year, month, day, hour
+
+user_agent = unquote(user_agent).replace('\t', ' ')
m = self.dateRe.match(dt)
if not m:
@@ -180,6 +182,9 @@
month = unicode(d.month)
day = unicode(d.day)
hour = unicode(d.hour)
+
+if self.xcsRe.match(x_analytics):
+x_analytics = 'zero=' + x_analytics
if 'zero=' not in x_analytics:
if defaultXcs:
@@ -207,19 +212,23 @@
continue
cache_status = httpStatuses[cache_status]
-m = self.urlRe.match(uri)
-if not m:
-safePrint(u'URL parsing failed: %s\n%s' % (uri, line))
-continue
-if m.group(1).lower() == u'https' and u'https=' not in
x_analytics:
-x_analytics += u'https=1'
-uri_host = m.group(2)
-if uri_host.endswith(':80'):
-uri_host = uri_host[:-3]
-if uri_host.endswith('.'):
-uri_host = uri_host[:-1]
-uri_path = m.group(3)
-uri_query = m.group(4)
+if uri == 'NONE://' or (http_method == 'CONNECT' and uri ==
':0'):
+uri_host = uri_path = uri_query = ''
+
+else:
+
[MediaWiki-commits] [Gerrit] Updated ZeroPortal to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/169922 Change subject: Updated ZeroPortal to master .. Updated ZeroPortal to master Change-Id: I8f8a48b21472fe22c4cf4be5f08a27154c6d798e --- M extensions/ZeroPortal 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/22/169922/1 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 7b0de38..3b71ad1 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 7b0de38e890649a8bbaebe4b977b862ccd74a5c8 +Subproject commit 3b71ad1590e0dbc1b23b6eb945734e69f2a83b14 -- To view, visit https://gerrit.wikimedia.org/r/169922 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I8f8a48b21472fe22c4cf4be5f08a27154c6d798e Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf6 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated ZeroPortal to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/169923 Change subject: Updated ZeroPortal to master .. Updated ZeroPortal to master Change-Id: I67f77da1245eb9362c7f776be70dc37bc84eda27 --- M extensions/ZeroPortal 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/23/169923/1 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 65cae35..3b71ad1 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 65cae35723ed75d3975b7d9d47256f5cb6ed60ff +Subproject commit 3b71ad1590e0dbc1b23b6eb945734e69f2a83b14 -- To view, visit https://gerrit.wikimedia.org/r/169923 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I67f77da1245eb9362c7f776be70dc37bc84eda27 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf5 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Revert Updated ZeroPortal to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/169929 Change subject: Revert Updated ZeroPortal to master .. Revert Updated ZeroPortal to master This reverts commit 9a33cbdbef08c1abb42f6d2b1167e643edfeb8e6. Change-Id: I74d701aaadcc92633613a6eb954190f8789d0cea --- M extensions/ZeroPortal 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/29/169929/1 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 3b71ad1..7b0de38 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 3b71ad1590e0dbc1b23b6eb945734e69f2a83b14 +Subproject commit 7b0de38e890649a8bbaebe4b977b862ccd74a5c8 -- To view, visit https://gerrit.wikimedia.org/r/169929 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I74d701aaadcc92633613a6eb954190f8789d0cea Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf6 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Revert Updated ZeroPortal to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/169928 Change subject: Revert Updated ZeroPortal to master .. Revert Updated ZeroPortal to master This reverts commit f641c3722ff822a725e35bec9721c5b3eb9c7899. Change-Id: I94d14b675096136fde2e5b6b9110c4d13bc5fe3d --- M extensions/ZeroPortal 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/28/169928/1 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 3b71ad1..65cae35 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 3b71ad1590e0dbc1b23b6eb945734e69f2a83b14 +Subproject commit 65cae35723ed75d3975b7d9d47256f5cb6ed60ff -- To view, visit https://gerrit.wikimedia.org/r/169928 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I94d14b675096136fde2e5b6b9110c4d13bc5fe3d Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf5 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] ZeroPortal setRawResult lua mode - change (mediawiki...ZeroPortal)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/169563
Change subject: ZeroPortal setRawResult lua mode
..
ZeroPortal setRawResult lua mode
This replaces the very hacky (and partially escaping broken) mode
where lua module decides to output data in the raw format.
Now, lua can set raw output data directly with
mw.zeroportal.setRawResult(text)
This function is only enabled when used from Special:ZeroPortal.
Eventually, we might want to port this to the scribunto ext.
A second patch will remove current raw=1 code once code is fixed.
Change-Id: Ibb97523f6b3164268eaa80d94ddb001e7f6a6088
---
M ZeroPortal.php
A includes/LuaLibrary.lua
A includes/LuaLibrary.php
M includes/PortalSpecialPage.php
4 files changed, 136 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroPortal
refs/changes/63/169563/1
diff --git a/ZeroPortal.php b/ZeroPortal.php
index 611d611..5f7056b 100644
--- a/ZeroPortal.php
+++ b/ZeroPortal.php
@@ -40,6 +40,7 @@
foreach ( array(
'ApiZeroPortal',
'ConfigPageHooks',
+ 'LuaLibrary',
'PortalSpecialPage',
'ZeroConfigView',
) as $key = $class ) {
@@ -64,6 +65,7 @@
$wgAPIModules['zeroportal'] = 'ZeroPortal\ApiZeroPortal';
$wgHooks['BeforePageDisplay'][] =
'ZeroPortal\ConfigPageHooks::onBeforePageDisplay';
+$wgHooks['ScribuntoExternalLibraries'][] =
'ZeroPortal\LuaLibrary::onScribuntoExternalLibraries';
// Set our own view class for the JsonZeroConfig model, and configure local
storage
$wgJsonConfigModels['JsonZeroConfig']['view'] = 'ZeroPortal\ZeroConfigView';
diff --git a/includes/LuaLibrary.lua b/includes/LuaLibrary.lua
new file mode 100644
index 000..2917e19
--- /dev/null
+++ b/includes/LuaLibrary.lua
@@ -0,0 +1,21 @@
+local p = {}
+local php
+
+function p.setupInterface( options )
+-- Boilerplate
+p.setupInterface = nil
+php = mw_interface
+mw_interface = nil
+
+-- Register this library in the mw global
+mw = mw or {}
+mw.zeroportal = p
+
+package.loaded['mw.zeroportal'] = p
+end
+
+function p.setRawResult( result )
+return php.setRawResult( result )
+end
+
+return p
diff --git a/includes/LuaLibrary.php b/includes/LuaLibrary.php
new file mode 100644
index 000..d4a8e87
--- /dev/null
+++ b/includes/LuaLibrary.php
@@ -0,0 +1,103 @@
+?php
+
+namespace ZeroPortal;
+
+use Scribunto_LuaError;
+use Scribunto_LuaLibraryBase;
+use IContextSource;
+
+class LuaLibrary extends Scribunto_LuaLibraryBase {
+
+ const luaNamespace = 'mw.zeroportal.';
+
+ /** @var IContextSource */
+ private static $context = null;
+
+ /** @var mixed */
+ private static $result = null;
+
+ /**
+* Add Lua library if the current title is our special module
+* @param string $engine
+* @param string[] $extraLibraries
+* @return bool
+*/
+ public static function onScribuntoExternalLibraries( $engine, array
$extraLibraries ) {
+ if ( $engine == 'lua' ) {
+ $extraLibraries['mw.zeroportal'] =
'ZeroPortal\LuaLibrary';
+ }
+ return true;
+ }
+
+ /**
+* @param IContextSource $context
+*/
+ public static function setContext( $context ) {
+ self::$context = $context;
+ }
+
+ /**
+* @return IContextSource
+* @throws Scribunto_LuaError
+*/
+ public static function getContext() {
+ if ( !self::$context ) {
+ // Something is seriously wrong - it shouldn't even be
possible to call this function
+ throw new Scribunto_LuaError( 'This function is only
available in Special:ZeroPortal' );
+ }
+ return self::$context;
+ }
+
+ /**
+* Finishes up the invocation, clean up context, and overrides output
if needed
+*/
+ public static function endInvoke() {
+ $ctx = self::getContext();
+ $out = $ctx-getOutput();
+ $req = $ctx-getRequest();
+ $resp = $req-response();
+ self::setContext( null ); // Ensure that subsequent lua
invocations don't permit
+
+ if ( !$req-wasPosted() ) {
+ $expiryUnixTime = time() + 300; // default 300 seconds
caching
+ $resp-header( 'Expires: ' . wfTimestamp( TS_RFC2822,
$expiryUnixTime ) );
+
+ $cacheMode = $ctx-getUser()-isAnon() ? 'public' :
'private';
+ $resp-header( 'Cache-Control: ' . $cacheMode . ',
must-revalidate, max-age=0' );
+ }
+
+ if ( self::$result ) {
+ $out-disable();
+ $resp-header( 'Content-Type: text/plain;
[MediaWiki-commits] [Gerrit] Change MIME type of the raw lua - change (mediawiki...ZeroPortal)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/169632
Change subject: Change MIME type of the raw lua
..
Change MIME type of the raw lua
Per comments in Ibb97523f6b3164268eaa80d94ddb001e7f6a6088, changing
mime type to be less likely to be mis-used/interpreted by the browser.
Change-Id: I02ed1f11b32e9e3a0bad3b5f8c7ad2689d34f53e
---
M includes/LuaLibrary.php
1 file changed, 1 insertion(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroPortal
refs/changes/32/169632/1
diff --git a/includes/LuaLibrary.php b/includes/LuaLibrary.php
index d4a8e87..d60ab96 100644
--- a/includes/LuaLibrary.php
+++ b/includes/LuaLibrary.php
@@ -68,7 +68,7 @@
if ( self::$result ) {
$out-disable();
- $resp-header( 'Content-Type: text/plain;
charset=UTF-8' );
+ $resp-header( 'Content-Type: application/json;
charset=UTF-8' );
echo self::$result;
return true;
} else {
--
To view, visit https://gerrit.wikimedia.org/r/169632
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I02ed1f11b32e9e3a0bad3b5f8c7ad2689d34f53e
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/ZeroPortal
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] log2dfs script - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/168980
Change subject: log2dfs script
..
log2dfs script
Change-Id: Iec7779d7392e338410755ba19358810d02d668a0
---
M scripts/clone-xcs.hql
A scripts/log2dfs.py
M scripts/run-hivezero.sh
3 files changed, 273 insertions(+), 29 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/80/168980/1
diff --git a/scripts/clone-xcs.hql b/scripts/clone-xcs.hql
index 5e7f500..39dc43c 100644
--- a/scripts/clone-xcs.hql
+++ b/scripts/clone-xcs.hql
@@ -5,35 +5,41 @@
-- Clone one day worth of data to a temp table
--
-- Usage:
--- hive -f clone-xcs.hql -d year=2014 -d month=9 -d day=15 -d xcs=515-05
-d table=tmp_clone
+-- hive -f clone-xcs.hql -d year=2014 -d month=10 -d day=11 -d hour=12 -d
xcs=515-05 -d table=tmp_clone
--
-- set hivevar:year=2014;
-- set hivevar:month=10;
--- set hivevar:day=21;
+-- set hivevar:day=11;
+-- set hivevar:hour=12;
-- set hivevar:xcs=515-05;
-- set hivevar:table=tmp_clone;
use yurik;
CREATE TABLE IF NOT EXISTS ${table} (
- hostname string,
- sequence bigint,
- dt string,
- time_firstbyte float,
- ip string,
- cache_status string,
- http_status string,
- response_size bigint,
- http_method string,
- uri_host string,
- uri_path string,
- uri_query string,
- content_type string,
- referer string,
- x_forwarded_for string,
- user_agent string,
- accept_language string,
- x_analytics string)
+ `hostname` string,
+ `sequence` bigint,
+ `dt` string,
+ `time_firstbyte` float,
+ `ip` string,
+ `cache_status` string,
+ `http_status` string,
+ `response_size` bigint,
+ `http_method` string,
+ `uri_host` string,
+ `uri_path` string,
+ `uri_query` string,
+ `content_type` string,
+ `referer` string,
+ `x_forwarded_for` string,
+ `user_agent` string,
+ `accept_language` string,
+ `x_analytics` string,
+ `webrequest_source` string,
+ `year` int,
+ `month` int,
+ `day` int,
+ `hour` int)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t';
@@ -47,6 +53,7 @@
AND year=${year}
AND month=${month}
AND day=${day}
+AND hour=${hour}
AND x_analytics LIKE '%zero=%'
AND SUBSTR(uri_path, 1, 6) = '/wiki/'
AND (
@@ -65,5 +72,5 @@
AND http_status NOT IN ( '301', '302', '303' )
AND uri_host RLIKE '^[A-Za-z0-9-]+(\\.(zero|m))?\\.[a-z]*\\.org$'
AND NOT (SPLIT(TRANSLATE(SUBSTR(uri_path, 7), ' ', '_'), '#')[0]
RLIKE '^[Uu]ndefined$')
-AND regexp_extract(x_analytics, 'zero=([^\;]+)') = ${xcs}
-
+AND regexp_extract(x_analytics, 'zero=([^\;]+)') = '${xcs}'
+;
diff --git a/scripts/log2dfs.py b/scripts/log2dfs.py
new file mode 100644
index 000..4050b46
--- /dev/null
+++ b/scripts/log2dfs.py
@@ -0,0 +1,236 @@
+# coding=utf-8
+import gzip
+import re
+
+try:
+from urllib.parse import unquote
+except ImportError:
+from urllib import unquote
+
+from logprocessor import *
+
+
+columnHdrCache =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
+columnHdrResult =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,iszero,ison,count'.split(',')
+validSubDomains = {'m', 'zero', 'mobile', 'wap'}
+validHttpCode = {'200', '304'}
+
+
+def isValidInt(val):
+try:
+int(val)
+return True
+except ValueError:
+return False
+
+
+xcsFromName = {
+'celcom-malaysia': '502-13',
+'dialog-sri-lanka': '413-02',
+'digi-malaysia': '502-16',
+'dtac-thailand': '520-18',
+'grameenphone-bangladesh': '470-01',
+'hello-cambodia': '456-02',
+'orange-botswana': '652-02',
+'orange-cameroon': '624-02',
+'orange-congo': '630-86',
+'orange-ivory-coast': '612-03',
+'orange-kenya': '639-07', # Also 639-02 (Safaricom Kenya)
+'orange-morocco': '604-00',
+'orange-niger': '614-04',
+'orange-tunesia': '605-01', # Also 639-02 (Safaricom Kenya)
+'orange-uganda': '641-14',
+'saudi-telecom': '420-01', # Also 652-02 (orange-botswana)
+'tata-india': '405-029',
+'telenor-montenegro': '297-01', # Also 250-99 (beeline ru)
+'tim-brasil': '724-02',
+'vodaphone-india': '404-01',
+'xl-axiata-indonesia': '510-11',
+}
+
+httpStatuses = {
+'-': '',
+'hit': 'hit',
+'miss': 'miss',
+'pass': 'pass',
+'TCP_CLIENT_REFRESH_MISS': 'miss',
+'TCP_MEM_HIT': 'hit',
+'TCP_MISS': 'miss',
+'TCP_REFRESH_HIT': 'hit',
+'FAKE_CACHE_STATUS': '',
+'TCP_HIT': 'hit',
+'TCP_IMS_HIT': 'hit',
+'TCP_DENIED': '',
+'TCP_REFRESH_MISS': 'miss',
+'TCP_NEGATIVE_HIT': 'hit',
+}
+
+
+class LogConverter(LogProcessor):
+def __init__(self, settingsFile='settings/log2dfs.json'):
+super(LogConverter, self).__init__(settingsFile, 'w2h')
+
+self.logFileRe = re.compile(r'\d\d\d\d\d\d\d\d')
+self.dateRe =
[MediaWiki-commits] [Gerrit] log2dfs script - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: log2dfs script
..
log2dfs script
Change-Id: Iec7779d7392e338410755ba19358810d02d668a0
---
M scripts/clone-xcs.hql
A scripts/log2dfs.py
M scripts/run-hivezero.sh
3 files changed, 276 insertions(+), 29 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/clone-xcs.hql b/scripts/clone-xcs.hql
index 5e7f500..39dc43c 100644
--- a/scripts/clone-xcs.hql
+++ b/scripts/clone-xcs.hql
@@ -5,35 +5,41 @@
-- Clone one day worth of data to a temp table
--
-- Usage:
--- hive -f clone-xcs.hql -d year=2014 -d month=9 -d day=15 -d xcs=515-05
-d table=tmp_clone
+-- hive -f clone-xcs.hql -d year=2014 -d month=10 -d day=11 -d hour=12 -d
xcs=515-05 -d table=tmp_clone
--
-- set hivevar:year=2014;
-- set hivevar:month=10;
--- set hivevar:day=21;
+-- set hivevar:day=11;
+-- set hivevar:hour=12;
-- set hivevar:xcs=515-05;
-- set hivevar:table=tmp_clone;
use yurik;
CREATE TABLE IF NOT EXISTS ${table} (
- hostname string,
- sequence bigint,
- dt string,
- time_firstbyte float,
- ip string,
- cache_status string,
- http_status string,
- response_size bigint,
- http_method string,
- uri_host string,
- uri_path string,
- uri_query string,
- content_type string,
- referer string,
- x_forwarded_for string,
- user_agent string,
- accept_language string,
- x_analytics string)
+ `hostname` string,
+ `sequence` bigint,
+ `dt` string,
+ `time_firstbyte` float,
+ `ip` string,
+ `cache_status` string,
+ `http_status` string,
+ `response_size` bigint,
+ `http_method` string,
+ `uri_host` string,
+ `uri_path` string,
+ `uri_query` string,
+ `content_type` string,
+ `referer` string,
+ `x_forwarded_for` string,
+ `user_agent` string,
+ `accept_language` string,
+ `x_analytics` string,
+ `webrequest_source` string,
+ `year` int,
+ `month` int,
+ `day` int,
+ `hour` int)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t';
@@ -47,6 +53,7 @@
AND year=${year}
AND month=${month}
AND day=${day}
+AND hour=${hour}
AND x_analytics LIKE '%zero=%'
AND SUBSTR(uri_path, 1, 6) = '/wiki/'
AND (
@@ -65,5 +72,5 @@
AND http_status NOT IN ( '301', '302', '303' )
AND uri_host RLIKE '^[A-Za-z0-9-]+(\\.(zero|m))?\\.[a-z]*\\.org$'
AND NOT (SPLIT(TRANSLATE(SUBSTR(uri_path, 7), ' ', '_'), '#')[0]
RLIKE '^[Uu]ndefined$')
-AND regexp_extract(x_analytics, 'zero=([^\;]+)') = ${xcs}
-
+AND regexp_extract(x_analytics, 'zero=([^\;]+)') = '${xcs}'
+;
diff --git a/scripts/log2dfs.py b/scripts/log2dfs.py
new file mode 100644
index 000..59aa394
--- /dev/null
+++ b/scripts/log2dfs.py
@@ -0,0 +1,239 @@
+# coding=utf-8
+import gzip
+import re
+import sys
+
+try:
+from urllib.parse import unquote
+except ImportError:
+from urllib import unquote
+
+from logprocessor import *
+
+
+columnHdrCache =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
+columnHdrResult =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,iszero,ison,count'.split(',')
+validSubDomains = {'m', 'zero', 'mobile', 'wap'}
+validHttpCode = {'200', '304'}
+
+
+def isValidInt(val):
+try:
+int(val)
+return True
+except ValueError:
+return False
+
+
+xcsFromName = {
+'celcom-malaysia': '502-13',
+'dialog-sri-lanka': '413-02',
+'digi-malaysia': '502-16',
+'dtac-thailand': '520-18',
+'grameenphone-bangladesh': '470-01',
+'hello-cambodia': '456-02',
+'orange-botswana': '652-02',
+'orange-cameroon': '624-02',
+'orange-congo': '630-86',
+'orange-ivory-coast': '612-03',
+'orange-kenya': '639-07', # Also 639-02 (Safaricom Kenya)
+'orange-morocco': '604-00',
+'orange-niger': '614-04',
+'orange-tunesia': '605-01', # Also 639-02 (Safaricom Kenya)
+'orange-uganda': '641-14',
+'saudi-telecom': '420-01', # Also 652-02 (orange-botswana)
+'tata-india': '405-029',
+'telenor-montenegro': '297-01', # Also 250-99 (beeline ru)
+'tim-brasil': '724-02',
+'vodaphone-india': '404-01',
+'xl-axiata-indonesia': '510-11',
+}
+
+httpStatuses = {
+'-': '',
+'hit': 'hit',
+'miss': 'miss',
+'pass': 'pass',
+'TCP_CLIENT_REFRESH_MISS': 'miss',
+'TCP_MEM_HIT': 'hit',
+'TCP_MISS': 'miss',
+'TCP_REFRESH_HIT': 'hit',
+'FAKE_CACHE_STATUS': '',
+'TCP_HIT': 'hit',
+'TCP_IMS_HIT': 'hit',
+'TCP_DENIED': '',
+'TCP_REFRESH_MISS': 'miss',
+'TCP_NEGATIVE_HIT': 'hit',
+}
+
+
+class LogConverter(LogProcessor):
+def __init__(self, settingsFile='settings/log2dfs.json',
logDatePattern=False):
+super(LogConverter, self).__init__(settingsFile, 'w2h')
+
+if not logDatePattern:
+logDatePattern = r'\d\d\d\d\d\d\d\d'
+self.logFileRe =
[MediaWiki-commits] [Gerrit] log2dfs fix pattern match picker - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/168986
Change subject: log2dfs fix pattern match picker
..
log2dfs fix pattern match picker
Change-Id: I698d2a5a0d2549dfda016ce712cebe2c88fbfcba
---
M scripts/log2dfs.py
M scripts/logprocessor.py
2 files changed, 9 insertions(+), 7 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/86/168986/1
diff --git a/scripts/log2dfs.py b/scripts/log2dfs.py
index 59aa394..a4d7b51 100644
--- a/scripts/log2dfs.py
+++ b/scripts/log2dfs.py
@@ -68,12 +68,12 @@
class LogConverter(LogProcessor):
-def __init__(self, settingsFile='settings/log2dfs.json',
logDatePattern=False):
+def __init__(self, filePattern=False,
settingsFile='settings/log2dfs.json'):
super(LogConverter, self).__init__(settingsFile, 'w2h')
-if not logDatePattern:
-logDatePattern = r'\d\d\d\d\d\d\d\d'
-self.logFileRe = re.compile(unicode(logDatePattern), re.IGNORECASE)
+if not filePattern:
+filePattern = r'\d\d\d\d\d\d\d\d'
+self.logFileRe = re.compile(unicode(filePattern), re.IGNORECASE)
self.dateRe = re.compile(r'(201\d-\d\d-\d\dT\d\d):\d\d:\d\d(\.\d+)?')
self.urlRe = re.compile(r'^(https?)://([^/]+)([^?#]*)(.*)',
re.IGNORECASE)
@@ -82,7 +82,7 @@
safePrint('Processing log files')
for f in os.listdir(self.pathLogs):
-if not self.logFileRe.match(f):
+if not self.logFileRe.search(f):
continue
logFile = os.path.join(self.pathLogs, f)
statFile = os.path.join(self.pathCache, f)
@@ -235,5 +235,5 @@
if __name__ == '__main__':
-# LogConverter(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
-LogConverter(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
+# LogConverter(filePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
+LogConverter(filePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
diff --git a/scripts/logprocessor.py b/scripts/logprocessor.py
index 36c7548..0e425b1 100644
--- a/scripts/logprocessor.py
+++ b/scripts/logprocessor.py
@@ -125,6 +125,8 @@
self.onSettingsLoaded()
def normalizePath(self, path, relToSettings=True):
+if not path:
+return False
if not os.path.isabs(path) and relToSettings:
path = os.path.join(os.path.dirname(self.settingsFile), path)
path = os.path.abspath(os.path.normpath(path))
--
To view, visit https://gerrit.wikimedia.org/r/168986
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I698d2a5a0d2549dfda016ce712cebe2c88fbfcba
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] log2dfs fix pattern match picker - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: log2dfs fix pattern match picker
..
log2dfs fix pattern match picker
Change-Id: I698d2a5a0d2549dfda016ce712cebe2c88fbfcba
---
M scripts/log2dfs.py
M scripts/logprocessor.py
2 files changed, 9 insertions(+), 7 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/log2dfs.py b/scripts/log2dfs.py
index 59aa394..a4d7b51 100644
--- a/scripts/log2dfs.py
+++ b/scripts/log2dfs.py
@@ -68,12 +68,12 @@
class LogConverter(LogProcessor):
-def __init__(self, settingsFile='settings/log2dfs.json',
logDatePattern=False):
+def __init__(self, filePattern=False,
settingsFile='settings/log2dfs.json'):
super(LogConverter, self).__init__(settingsFile, 'w2h')
-if not logDatePattern:
-logDatePattern = r'\d\d\d\d\d\d\d\d'
-self.logFileRe = re.compile(unicode(logDatePattern), re.IGNORECASE)
+if not filePattern:
+filePattern = r'\d\d\d\d\d\d\d\d'
+self.logFileRe = re.compile(unicode(filePattern), re.IGNORECASE)
self.dateRe = re.compile(r'(201\d-\d\d-\d\dT\d\d):\d\d:\d\d(\.\d+)?')
self.urlRe = re.compile(r'^(https?)://([^/]+)([^?#]*)(.*)',
re.IGNORECASE)
@@ -82,7 +82,7 @@
safePrint('Processing log files')
for f in os.listdir(self.pathLogs):
-if not self.logFileRe.match(f):
+if not self.logFileRe.search(f):
continue
logFile = os.path.join(self.pathLogs, f)
statFile = os.path.join(self.pathCache, f)
@@ -235,5 +235,5 @@
if __name__ == '__main__':
-# LogConverter(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
-LogConverter(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
+# LogConverter(filePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
+LogConverter(filePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
diff --git a/scripts/logprocessor.py b/scripts/logprocessor.py
index 36c7548..0e425b1 100644
--- a/scripts/logprocessor.py
+++ b/scripts/logprocessor.py
@@ -125,6 +125,8 @@
self.onSettingsLoaded()
def normalizePath(self, path, relToSettings=True):
+if not path:
+return False
if not os.path.isabs(path) and relToSettings:
path = os.path.join(os.path.dirname(self.settingsFile), path)
path = os.path.abspath(os.path.normpath(path))
--
To view, visit https://gerrit.wikimedia.org/r/168986
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I698d2a5a0d2549dfda016ce712cebe2c88fbfcba
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
Gerrit-Reviewer: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] clone script, overwrite error, mem error - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/168333
Change subject: clone script, overwrite error, mem error
..
clone script, overwrite error, mem error
Change-Id: Icc87571d1d897969184516c4dc0ea5b9cfdcdf63
---
A scripts/clone-xcs.hql
M scripts/run-hivezero.sh
M scripts/weblogs2.py
M scripts/zero-counts.hql
4 files changed, 88 insertions(+), 10 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/33/168333/1
diff --git a/scripts/clone-xcs.hql b/scripts/clone-xcs.hql
new file mode 100644
index 000..5e7f500
--- /dev/null
+++ b/scripts/clone-xcs.hql
@@ -0,0 +1,69 @@
+set hive.exec.dynamic.partition.mode=nonstrict;
+SET hive.exec.compress.output=false;
+--^ To work around HIVE-3296, we have SETs before any comments
+
+-- Clone one day worth of data to a temp table
+--
+-- Usage:
+-- hive -f clone-xcs.hql -d year=2014 -d month=9 -d day=15 -d xcs=515-05
-d table=tmp_clone
+--
+-- set hivevar:year=2014;
+-- set hivevar:month=10;
+-- set hivevar:day=21;
+-- set hivevar:xcs=515-05;
+-- set hivevar:table=tmp_clone;
+
+use yurik;
+
+CREATE TABLE IF NOT EXISTS ${table} (
+ hostname string,
+ sequence bigint,
+ dt string,
+ time_firstbyte float,
+ ip string,
+ cache_status string,
+ http_status string,
+ response_size bigint,
+ http_method string,
+ uri_host string,
+ uri_path string,
+ uri_query string,
+ content_type string,
+ referer string,
+ x_forwarded_for string,
+ user_agent string,
+ accept_language string,
+ x_analytics string)
+ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '\t';
+
+INSERT OVERWRITE TABLE ${table}
+
+SELECT
+*
+FROM wmf_raw.webrequest
+WHERE
+webrequest_source IN ('text', 'mobile')
+AND year=${year}
+AND month=${month}
+AND day=${day}
+AND x_analytics LIKE '%zero=%'
+AND SUBSTR(uri_path, 1, 6) = '/wiki/'
+AND (
+(
+SUBSTR(ip, 1, 9) != '10.128.0.'
+AND SUBSTR(ip, 1, 11) NOT IN (
+'208.80.152.',
+'208.80.153.',
+'208.80.154.',
+'208.80.155.',
+'91.198.174.'
+)
+) OR x_forwarded_for != '-'
+)
+AND SUBSTR(uri_path, 1, 31) != '/wiki/Special:CentralAutoLogin/'
+AND http_status NOT IN ( '301', '302', '303' )
+AND uri_host RLIKE '^[A-Za-z0-9-]+(\\.(zero|m))?\\.[a-z]*\\.org$'
+AND NOT (SPLIT(TRANSLATE(SUBSTR(uri_path, 7), ' ', '_'), '#')[0]
RLIKE '^[Uu]ndefined$')
+AND regexp_extract(x_analytics, 'zero=([^\;]+)') = ${xcs}
+
diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh
index e04685f..0df0a65 100755
--- a/scripts/run-hivezero.sh
+++ b/scripts/run-hivezero.sh
@@ -9,5 +9,5 @@
for ((day = $3; day = $last; day++)); do
printf -v p %04d-%02d-%02d $1 $2 $day
echo hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$day
-d date=$p
- hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$day -d
date=$p
+ export HADOOP_HEAPSIZE=1024 hive -f zero-counts.hql -d year=$1 -d
month=$2 -d day=$day -d date=$p
done
diff --git a/scripts/weblogs2.py b/scripts/weblogs2.py
index 2432422..0812611 100644
--- a/scripts/weblogs2.py
+++ b/scripts/weblogs2.py
@@ -2,6 +2,7 @@
import StringIO
import re
import collections
+
from pandas import read_table, pivot_table
from pandas.core.frame import DataFrame, Series
import numpy as np
@@ -36,7 +37,7 @@
if self.settings.pathCacheLegacy:
self.pathCacheLegacy =
self.normalizePath(self.settings.pathCacheLegacy)
else:
-self.pathCacheLegacy = self.settings.pathCacheLegacy
+self.pathCacheLegacy = False
self.legacyFileRe =
re.compile(r'^(zero\.tsv\.log-(\d+)\.gz)__\d+\.tsv$', re.IGNORECASE)
@@ -86,6 +87,8 @@
else:
raise ValueError('Unrecognized key (%s) in file %s' %
(joinValues(vals), f))
(dt, typ, xcs, via, ipset, https, lang, subdomain, site,
count) = vals
+
+via = via.upper()
error = False
if xcs == '404-01b':
@@ -151,7 +154,8 @@
# 0 1 2 34 56 7
# 250-99 DIRECT default http ru zero wikipedia 1000
(xcs, via, ipset, https, lang, subdomain, site, count) =
vals
-via = via if via else u'DIRECT'
+
+via = via.upper() if via else u'DIRECT'
ipset = ipset if ipset else u'default'
https = https if https else u'http'
@@ -195,9 +199,9 @@
if legacyStats:
[MediaWiki-commits] [Gerrit] clone script, overwrite error, mem error - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: clone script, overwrite error, mem error
..
clone script, overwrite error, mem error
Change-Id: Icc87571d1d897969184516c4dc0ea5b9cfdcdf63
---
A scripts/clone-xcs.hql
M scripts/run-hivezero.sh
M scripts/weblogs2.py
M scripts/zero-counts.hql
4 files changed, 88 insertions(+), 10 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/clone-xcs.hql b/scripts/clone-xcs.hql
new file mode 100644
index 000..5e7f500
--- /dev/null
+++ b/scripts/clone-xcs.hql
@@ -0,0 +1,69 @@
+set hive.exec.dynamic.partition.mode=nonstrict;
+SET hive.exec.compress.output=false;
+--^ To work around HIVE-3296, we have SETs before any comments
+
+-- Clone one day worth of data to a temp table
+--
+-- Usage:
+-- hive -f clone-xcs.hql -d year=2014 -d month=9 -d day=15 -d xcs=515-05
-d table=tmp_clone
+--
+-- set hivevar:year=2014;
+-- set hivevar:month=10;
+-- set hivevar:day=21;
+-- set hivevar:xcs=515-05;
+-- set hivevar:table=tmp_clone;
+
+use yurik;
+
+CREATE TABLE IF NOT EXISTS ${table} (
+ hostname string,
+ sequence bigint,
+ dt string,
+ time_firstbyte float,
+ ip string,
+ cache_status string,
+ http_status string,
+ response_size bigint,
+ http_method string,
+ uri_host string,
+ uri_path string,
+ uri_query string,
+ content_type string,
+ referer string,
+ x_forwarded_for string,
+ user_agent string,
+ accept_language string,
+ x_analytics string)
+ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '\t';
+
+INSERT OVERWRITE TABLE ${table}
+
+SELECT
+*
+FROM wmf_raw.webrequest
+WHERE
+webrequest_source IN ('text', 'mobile')
+AND year=${year}
+AND month=${month}
+AND day=${day}
+AND x_analytics LIKE '%zero=%'
+AND SUBSTR(uri_path, 1, 6) = '/wiki/'
+AND (
+(
+SUBSTR(ip, 1, 9) != '10.128.0.'
+AND SUBSTR(ip, 1, 11) NOT IN (
+'208.80.152.',
+'208.80.153.',
+'208.80.154.',
+'208.80.155.',
+'91.198.174.'
+)
+) OR x_forwarded_for != '-'
+)
+AND SUBSTR(uri_path, 1, 31) != '/wiki/Special:CentralAutoLogin/'
+AND http_status NOT IN ( '301', '302', '303' )
+AND uri_host RLIKE '^[A-Za-z0-9-]+(\\.(zero|m))?\\.[a-z]*\\.org$'
+AND NOT (SPLIT(TRANSLATE(SUBSTR(uri_path, 7), ' ', '_'), '#')[0]
RLIKE '^[Uu]ndefined$')
+AND regexp_extract(x_analytics, 'zero=([^\;]+)') = ${xcs}
+
diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh
index e04685f..0df0a65 100755
--- a/scripts/run-hivezero.sh
+++ b/scripts/run-hivezero.sh
@@ -9,5 +9,5 @@
for ((day = $3; day = $last; day++)); do
printf -v p %04d-%02d-%02d $1 $2 $day
echo hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$day
-d date=$p
- hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$day -d
date=$p
+ export HADOOP_HEAPSIZE=1024 hive -f zero-counts.hql -d year=$1 -d
month=$2 -d day=$day -d date=$p
done
diff --git a/scripts/weblogs2.py b/scripts/weblogs2.py
index 2432422..0812611 100644
--- a/scripts/weblogs2.py
+++ b/scripts/weblogs2.py
@@ -2,6 +2,7 @@
import StringIO
import re
import collections
+
from pandas import read_table, pivot_table
from pandas.core.frame import DataFrame, Series
import numpy as np
@@ -36,7 +37,7 @@
if self.settings.pathCacheLegacy:
self.pathCacheLegacy =
self.normalizePath(self.settings.pathCacheLegacy)
else:
-self.pathCacheLegacy = self.settings.pathCacheLegacy
+self.pathCacheLegacy = False
self.legacyFileRe =
re.compile(r'^(zero\.tsv\.log-(\d+)\.gz)__\d+\.tsv$', re.IGNORECASE)
@@ -86,6 +87,8 @@
else:
raise ValueError('Unrecognized key (%s) in file %s' %
(joinValues(vals), f))
(dt, typ, xcs, via, ipset, https, lang, subdomain, site,
count) = vals
+
+via = via.upper()
error = False
if xcs == '404-01b':
@@ -151,7 +154,8 @@
# 0 1 2 34 56 7
# 250-99 DIRECT default http ru zero wikipedia 1000
(xcs, via, ipset, https, lang, subdomain, site, count) =
vals
-via = via if via else u'DIRECT'
+
+via = via.upper() if via else u'DIRECT'
ipset = ipset if ipset else u'default'
https = https if https else u'http'
@@ -195,9 +199,9 @@
if legacyStats:
# Only add legacy data for dates that we haven't seen in hadoop
[MediaWiki-commits] [Gerrit] Adapted analytics pageview counter - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/168057
Change subject: Adapted analytics pageview counter
..
Adapted analytics pageview counter
Change-Id: I342b1360034c4f11c0c0f9ad2d964b43f890c0c4
---
A scripts/zero-counts.hql
1 file changed, 65 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/57/168057/1
diff --git a/scripts/zero-counts.hql b/scripts/zero-counts.hql
new file mode 100644
index 000..170cca5
--- /dev/null
+++ b/scripts/zero-counts.hql
@@ -0,0 +1,65 @@
+set hive.exec.dynamic.partition.mode=nonstrict;
+SET hive.exec.compress.output=true;
+--^ To work around HIVE-3296, we have SETs before any comments
+
+-- Extracts zero stats from webrequests into a separate table
+--
+-- Usage:
+-- hive -f zero-counts.hql \
+-- -d year=2014 \
+-- -d month=9 \
+-- -d day=15 \
+-- -d hour=20
+--
+
+-- set hivevar:year=2014;
+-- set hivevar:month=10;
+-- set hivevar:day=21;
+-- set hivevar:hour=1;
+-- printf('%d-%02d-%02d', ${year}, ${month}, ${day}) date,
+;
+
+INSERT OVERWRITE TABLE yurik.zero_webstats
+PARTITION(date)
+SELECT
+xcs, via, ipset, https, lang, subdomain, site, COUNT(*) count, date
+FROM (
+SELECT
+printf('%d-%02d-%02d', ${year}, ${month}, ${day}) date,
+regexp_extract(x_analytics, 'zero=([^\;]+)') xcs,
+regexp_extract(x_analytics, 'proxy=([^\;]+)') via,
+regexp_extract(x_analytics, 'zeronet=([^\;]+)') ipset,
+if (x_analytics LIKE '%https=1%', 'https', '') https,
+regexp_extract(uri_host,
'^([A-Za-z0-9-]+)(\\.(zero|m))?\\.([a-z]*)\\.org$', 1) lang,
+regexp_extract(uri_host,
'^([A-Za-z0-9-]+)(\\.(zero|m))?\\.([a-z]*)\\.org$', 3) subdomain,
+regexp_extract(uri_host,
'^([A-Za-z0-9-]+)(\\.(zero|m))?\\.([a-z]*)\\.org$', 4) site
+
+FROM wmf_raw.webrequest
+WHERE
+webrequest_source IN ('text', 'mobile')
+AND year=${year}
+AND month=${month}
+AND day=${day}
+AND hour=2
+AND x_analytics LIKE '%zero=%'
+AND SUBSTR(uri_path, 1, 6) = '/wiki/'
+AND (
+(
+SUBSTR(ip, 1, 9) != '10.128.0.'
+AND SUBSTR(ip, 1, 11) NOT IN (
+'208.80.152.',
+'208.80.153.',
+'208.80.154.',
+'208.80.155.',
+'91.198.174.'
+)
+) OR x_forwarded_for != '-'
+)
+AND SUBSTR(uri_path, 1, 31) != '/wiki/Special:CentralAutoLogin/'
+AND http_status NOT IN ( '301', '302', '303' )
+AND uri_host RLIKE '^[A-Za-z0-9-]+(\\.(zero|m))?\\.[a-z]*\\.org$'
+AND NOT (SPLIT(TRANSLATE(SUBSTR(uri_path, 7), ' ', '_'), '#')[0]
RLIKE '^[Uu]ndefined$')
+
+) prepared
+GROUP BY date, xcs, via, ipset, https, lang, subdomain, site
+DISTRIBUTE BY date;
--
To view, visit https://gerrit.wikimedia.org/r/168057
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I342b1360034c4f11c0c0f9ad2d964b43f890c0c4
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Adapted analytics pageview counter - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: Adapted analytics pageview counter
..
Adapted analytics pageview counter
Change-Id: I342b1360034c4f11c0c0f9ad2d964b43f890c0c4
---
A scripts/zero-counts.hql
1 file changed, 66 insertions(+), 0 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/zero-counts.hql b/scripts/zero-counts.hql
new file mode 100644
index 000..4c1b735
--- /dev/null
+++ b/scripts/zero-counts.hql
@@ -0,0 +1,66 @@
+set hive.exec.dynamic.partition.mode=nonstrict;
+SET hive.exec.compress.output=false;
+--^ To work around HIVE-3296, we have SETs before any comments
+
+-- Extracts zero stats from webrequests into a separate table
+--
+-- Usage:
+-- hive -f zero-counts.hql \
+-- -d year=2014 \
+-- -d month=9 \
+-- -d day=15 \
+-- -d hour=20
+--
+
+-- set hivevar:year=2014;
+-- set hivevar:month=10;
+-- set hivevar:day=21;
+-- set hivevar:hour=1;
+-- printf('%d-%02d-%02d', ${year}, ${month}, ${day}) date,
+-- ;
+-- AND hour=2
+
+
+INSERT OVERWRITE TABLE yurik.zero_webstats
+PARTITION(date)
+SELECT
+xcs, via, ipset, https, lang, subdomain, site, COUNT(*) count, date
+FROM (
+SELECT
+printf('%d-%02d-%02d', ${year}, ${month}, ${day}) date,
+regexp_extract(x_analytics, 'zero=([^\;]+)') xcs,
+regexp_extract(x_analytics, 'proxy=([^\;]+)') via,
+regexp_extract(x_analytics, 'zeronet=([^\;]+)') ipset,
+if (x_analytics LIKE '%https=1%', 'https', '') https,
+regexp_extract(uri_host,
'^([A-Za-z0-9-]+)(\\.(zero|m))?\\.([a-z]*)\\.org$', 1) lang,
+regexp_extract(uri_host,
'^([A-Za-z0-9-]+)(\\.(zero|m))?\\.([a-z]*)\\.org$', 3) subdomain,
+regexp_extract(uri_host,
'^([A-Za-z0-9-]+)(\\.(zero|m))?\\.([a-z]*)\\.org$', 4) site
+
+FROM wmf_raw.webrequest
+WHERE
+webrequest_source IN ('text', 'mobile')
+AND year=${year}
+AND month=${month}
+AND day=${day}
+AND x_analytics LIKE '%zero=%'
+AND SUBSTR(uri_path, 1, 6) = '/wiki/'
+AND (
+(
+SUBSTR(ip, 1, 9) != '10.128.0.'
+AND SUBSTR(ip, 1, 11) NOT IN (
+'208.80.152.',
+'208.80.153.',
+'208.80.154.',
+'208.80.155.',
+'91.198.174.'
+)
+) OR x_forwarded_for != '-'
+)
+AND SUBSTR(uri_path, 1, 31) != '/wiki/Special:CentralAutoLogin/'
+AND http_status NOT IN ( '301', '302', '303' )
+AND uri_host RLIKE '^[A-Za-z0-9-]+(\\.(zero|m))?\\.[a-z]*\\.org$'
+AND NOT (SPLIT(TRANSLATE(SUBSTR(uri_path, 7), ' ', '_'), '#')[0]
RLIKE '^[Uu]ndefined$')
+
+) prepared
+GROUP BY date, xcs, via, ipset, https, lang, subdomain, site
+DISTRIBUTE BY date;
--
To view, visit https://gerrit.wikimedia.org/r/168057
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I342b1360034c4f11c0c0f9ad2d964b43f890c0c4
Gerrit-PatchSet: 2
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
Gerrit-Reviewer: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] don't overwrite if exists - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: don't overwrite if exists
..
don't overwrite if exists
Change-Id: I0edbc94c61395f28b6c72a19f47656e654cfc773
---
M scripts/zero-counts.hql
1 file changed, 6 insertions(+), 10 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/zero-counts.hql b/scripts/zero-counts.hql
index 4c1b735..67a7fb6 100644
--- a/scripts/zero-counts.hql
+++ b/scripts/zero-counts.hql
@@ -5,24 +5,20 @@
-- Extracts zero stats from webrequests into a separate table
--
-- Usage:
--- hive -f zero-counts.hql \
--- -d year=2014 \
--- -d month=9 \
--- -d day=15 \
--- -d hour=20
+-- hive -f zero-counts.hql -d year=2014 -d month=9 -d day=15
--
+-- Range usage:
+-- cat inp.txt | xargs -I % hive -f zero-counts.hql -d year=2014 -d
month=10 -d day=%
+-- or
+-- for i in {1..5}; do hive -f zero-counts.hql -d year=2014 -d month=10
-d day=$i; done
-- set hivevar:year=2014;
-- set hivevar:month=10;
-- set hivevar:day=21;
--- set hivevar:hour=1;
--- printf('%d-%02d-%02d', ${year}, ${month}, ${day}) date,
--- ;
--- AND hour=2
INSERT OVERWRITE TABLE yurik.zero_webstats
-PARTITION(date)
+PARTITION(date) IF NOT EXISTS
SELECT
xcs, via, ipset, https, lang, subdomain, site, COUNT(*) count, date
FROM (
--
To view, visit https://gerrit.wikimedia.org/r/168121
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0edbc94c61395f28b6c72a19f47656e654cfc773
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
Gerrit-Reviewer: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] don't overwrite if exists - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/168121
Change subject: don't overwrite if exists
..
don't overwrite if exists
Change-Id: I0edbc94c61395f28b6c72a19f47656e654cfc773
---
M scripts/zero-counts.hql
1 file changed, 6 insertions(+), 10 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/21/168121/1
diff --git a/scripts/zero-counts.hql b/scripts/zero-counts.hql
index 4c1b735..67a7fb6 100644
--- a/scripts/zero-counts.hql
+++ b/scripts/zero-counts.hql
@@ -5,24 +5,20 @@
-- Extracts zero stats from webrequests into a separate table
--
-- Usage:
--- hive -f zero-counts.hql \
--- -d year=2014 \
--- -d month=9 \
--- -d day=15 \
--- -d hour=20
+-- hive -f zero-counts.hql -d year=2014 -d month=9 -d day=15
--
+-- Range usage:
+-- cat inp.txt | xargs -I % hive -f zero-counts.hql -d year=2014 -d
month=10 -d day=%
+-- or
+-- for i in {1..5}; do hive -f zero-counts.hql -d year=2014 -d month=10
-d day=$i; done
-- set hivevar:year=2014;
-- set hivevar:month=10;
-- set hivevar:day=21;
--- set hivevar:hour=1;
--- printf('%d-%02d-%02d', ${year}, ${month}, ${day}) date,
--- ;
--- AND hour=2
INSERT OVERWRITE TABLE yurik.zero_webstats
-PARTITION(date)
+PARTITION(date) IF NOT EXISTS
SELECT
xcs, via, ipset, https, lang, subdomain, site, COUNT(*) count, date
FROM (
--
To view, visit https://gerrit.wikimedia.org/r/168121
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0edbc94c61395f28b6c72a19f47656e654cfc773
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] create table if doesn't exist, runner script - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/168168
Change subject: create table if doesn't exist, runner script
..
create table if doesn't exist, runner script
Change-Id: I29f04352801dabc312d08664e9521ab115d5e498
---
A scripts/run-hivezero.sh
M scripts/zero-counts.hql
2 files changed, 64 insertions(+), 11 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/68/168168/1
diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh
new file mode 100644
index 000..c1a8e96
--- /dev/null
+++ b/scripts/run-hivezero.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [[ -z $4 ]]; then
+ last=$3
+else
+ last=$4
+fi
+
+for ((day = $3; day = $last; day++)); do
+ printf -v p %04d-%02d-%02d $1 $2 $day
+ echo hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$3 -d
date=$p
+ hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$3 -d
date=$p
+done
diff --git a/scripts/zero-counts.hql b/scripts/zero-counts.hql
index 67a7fb6..2e7d0db 100644
--- a/scripts/zero-counts.hql
+++ b/scripts/zero-counts.hql
@@ -5,25 +5,38 @@
-- Extracts zero stats from webrequests into a separate table
--
-- Usage:
--- hive -f zero-counts.hql -d year=2014 -d month=9 -d day=15
+-- hive -f zero-counts.hql -d year=2014 -d month=9 -d day=15 -d
date=2014-09-15
+-- Date is duplicated because I haven't figured an easy way to set
date=printf()
--
--- Range usage:
--- cat inp.txt | xargs -I % hive -f zero-counts.hql -d year=2014 -d
month=10 -d day=%
--- or
--- for i in {1..5}; do hive -f zero-counts.hql -d year=2014 -d month=10
-d day=$i; done
-
-- set hivevar:year=2014;
-- set hivevar:month=10;
-- set hivevar:day=21;
+-- set hivevar:date=2014-10-21;
+
+use yurik;
+
+
+CREATE TABLE IF NOT EXISTS yurik.zero_webstats (
+ xcs string,
+ via string,
+ ipset string,
+ https string,
+ lang string,
+ subdomain string,
+ site string,
+ count bigint)
+PARTITIONED BY (
+ date string)
+ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '\t';
INSERT OVERWRITE TABLE yurik.zero_webstats
-PARTITION(date) IF NOT EXISTS
+PARTITION(date=${date}) IF NOT EXISTS
SELECT
-xcs, via, ipset, https, lang, subdomain, site, COUNT(*) count, date
+xcs, via, ipset, https, lang, subdomain, site, COUNT(*) count
FROM (
SELECT
-printf('%d-%02d-%02d', ${year}, ${month}, ${day}) date,
regexp_extract(x_analytics, 'zero=([^\;]+)') xcs,
regexp_extract(x_analytics, 'proxy=([^\;]+)') via,
regexp_extract(x_analytics, 'zeronet=([^\;]+)') ipset,
@@ -58,5 +71,32 @@
AND NOT (SPLIT(TRANSLATE(SUBSTR(uri_path, 7), ' ', '_'), '#')[0]
RLIKE '^[Uu]ndefined$')
) prepared
-GROUP BY date, xcs, via, ipset, https, lang, subdomain, site
-DISTRIBUTE BY date;
+GROUP BY xcs, via, ipset, https, lang, subdomain, site
+DISTRIBUTE BY printf('%d-%02d-%02d', ${year}, ${month}, ${day});
+
+
+-- CREATE TABLE IF NOT EXISTS yurik.zero_webstats_sum (
+-- date string,
+-- tag string,
+-- count bigint)
+-- PARTITIONED BY (
+-- xcs string)
+-- ROW FORMAT DELIMITED
+-- FIELDS TERMINATED BY '\t';
+
+-- xcs string,
+-- via string,
+-- ipset string,
+-- https string,
+-- lang string,
+-- subdomain string,
+-- site string,
+-- count bigint)
+
+-- INSERT OVERWRITE TABLE yurik.zero_webstats_sum
+-- PARTITION(xcs)
+-- SELECT
+-- date, tag, COUNT(*) count, xcs
+-- FROM yurik.zero_webstats
+-- GROUP BY xcs, date, tag
+-- DISTRIBUTE BY xcs;
--
To view, visit https://gerrit.wikimedia.org/r/168168
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I29f04352801dabc312d08664e9521ab115d5e498
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] create table if doesn't exist, runner script - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: create table if doesn't exist, runner script
..
create table if doesn't exist, runner script
Change-Id: I29f04352801dabc312d08664e9521ab115d5e498
---
A scripts/run-hivezero.sh
M scripts/zero-counts.hql
2 files changed, 64 insertions(+), 11 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh
new file mode 100644
index 000..c1a8e96
--- /dev/null
+++ b/scripts/run-hivezero.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [[ -z $4 ]]; then
+ last=$3
+else
+ last=$4
+fi
+
+for ((day = $3; day = $last; day++)); do
+ printf -v p %04d-%02d-%02d $1 $2 $day
+ echo hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$3 -d
date=$p
+ hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$3 -d
date=$p
+done
diff --git a/scripts/zero-counts.hql b/scripts/zero-counts.hql
index 67a7fb6..2e7d0db 100644
--- a/scripts/zero-counts.hql
+++ b/scripts/zero-counts.hql
@@ -5,25 +5,38 @@
-- Extracts zero stats from webrequests into a separate table
--
-- Usage:
--- hive -f zero-counts.hql -d year=2014 -d month=9 -d day=15
+-- hive -f zero-counts.hql -d year=2014 -d month=9 -d day=15 -d
date=2014-09-15
+-- Date is duplicated because I haven't figured an easy way to set
date=printf()
--
--- Range usage:
--- cat inp.txt | xargs -I % hive -f zero-counts.hql -d year=2014 -d
month=10 -d day=%
--- or
--- for i in {1..5}; do hive -f zero-counts.hql -d year=2014 -d month=10
-d day=$i; done
-
-- set hivevar:year=2014;
-- set hivevar:month=10;
-- set hivevar:day=21;
+-- set hivevar:date=2014-10-21;
+
+use yurik;
+
+
+CREATE TABLE IF NOT EXISTS yurik.zero_webstats (
+ xcs string,
+ via string,
+ ipset string,
+ https string,
+ lang string,
+ subdomain string,
+ site string,
+ count bigint)
+PARTITIONED BY (
+ date string)
+ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '\t';
INSERT OVERWRITE TABLE yurik.zero_webstats
-PARTITION(date) IF NOT EXISTS
+PARTITION(date=${date}) IF NOT EXISTS
SELECT
-xcs, via, ipset, https, lang, subdomain, site, COUNT(*) count, date
+xcs, via, ipset, https, lang, subdomain, site, COUNT(*) count
FROM (
SELECT
-printf('%d-%02d-%02d', ${year}, ${month}, ${day}) date,
regexp_extract(x_analytics, 'zero=([^\;]+)') xcs,
regexp_extract(x_analytics, 'proxy=([^\;]+)') via,
regexp_extract(x_analytics, 'zeronet=([^\;]+)') ipset,
@@ -58,5 +71,32 @@
AND NOT (SPLIT(TRANSLATE(SUBSTR(uri_path, 7), ' ', '_'), '#')[0]
RLIKE '^[Uu]ndefined$')
) prepared
-GROUP BY date, xcs, via, ipset, https, lang, subdomain, site
-DISTRIBUTE BY date;
+GROUP BY xcs, via, ipset, https, lang, subdomain, site
+DISTRIBUTE BY printf('%d-%02d-%02d', ${year}, ${month}, ${day});
+
+
+-- CREATE TABLE IF NOT EXISTS yurik.zero_webstats_sum (
+-- date string,
+-- tag string,
+-- count bigint)
+-- PARTITIONED BY (
+-- xcs string)
+-- ROW FORMAT DELIMITED
+-- FIELDS TERMINATED BY '\t';
+
+-- xcs string,
+-- via string,
+-- ipset string,
+-- https string,
+-- lang string,
+-- subdomain string,
+-- site string,
+-- count bigint)
+
+-- INSERT OVERWRITE TABLE yurik.zero_webstats_sum
+-- PARTITION(xcs)
+-- SELECT
+-- date, tag, COUNT(*) count, xcs
+-- FROM yurik.zero_webstats
+-- GROUP BY xcs, date, tag
+-- DISTRIBUTE BY xcs;
--
To view, visit https://gerrit.wikimedia.org/r/168168
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I29f04352801dabc312d08664e9521ab115d5e498
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
Gerrit-Reviewer: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] chmod +x run-hivezero.sh - change (analytics/zero-sms)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/168172 Change subject: chmod +x run-hivezero.sh .. chmod +x run-hivezero.sh Change-Id: I4aae2a13de2ba2b4a4b4e3130744de279a11a285 --- M scripts/run-hivezero.sh 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms refs/changes/72/168172/1 diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh old mode 100644 new mode 100755 -- To view, visit https://gerrit.wikimedia.org/r/168172 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I4aae2a13de2ba2b4a4b4e3130744de279a11a285 Gerrit-PatchSet: 1 Gerrit-Project: analytics/zero-sms Gerrit-Branch: master Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] chmod +x run-hivezero.sh - change (analytics/zero-sms)
Yurik has submitted this change and it was merged. Change subject: chmod +x run-hivezero.sh .. chmod +x run-hivezero.sh Change-Id: I4aae2a13de2ba2b4a4b4e3130744de279a11a285 --- M scripts/run-hivezero.sh 1 file changed, 0 insertions(+), 0 deletions(-) Approvals: Yurik: Verified; Looks good to me, approved diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh old mode 100644 new mode 100755 -- To view, visit https://gerrit.wikimedia.org/r/168172 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I4aae2a13de2ba2b4a4b4e3130744de279a11a285 Gerrit-PatchSet: 1 Gerrit-Project: analytics/zero-sms Gerrit-Branch: master Gerrit-Owner: Yurik yu...@wikimedia.org Gerrit-Reviewer: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] shell bug, weblogs2 to combine multi-sourced data - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/168195
Change subject: shell bug, weblogs2 to combine multi-sourced data
..
shell bug, weblogs2 to combine multi-sourced data
Change-Id: Idc09a1ee5434a8e134530cbb722f25430efff6b8
---
M scripts/run-hivezero.sh
A scripts/weblogs2.py
M scripts/zero-counts.hql
3 files changed, 301 insertions(+), 29 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/95/168195/1
diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh
index c1a8e96..e04685f 100755
--- a/scripts/run-hivezero.sh
+++ b/scripts/run-hivezero.sh
@@ -8,6 +8,6 @@
for ((day = $3; day = $last; day++)); do
printf -v p %04d-%02d-%02d $1 $2 $day
- echo hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$3 -d
date=$p
- hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$3 -d
date=$p
+ echo hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$day
-d date=$p
+ hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$day -d
date=$p
done
diff --git a/scripts/weblogs2.py b/scripts/weblogs2.py
new file mode 100644
index 000..2432422
--- /dev/null
+++ b/scripts/weblogs2.py
@@ -0,0 +1,299 @@
+# coding=utf-8
+import StringIO
+import re
+import collections
+from pandas import read_table, pivot_table
+from pandas.core.frame import DataFrame, Series
+import numpy as np
+
+from logprocessor import *
+
+
+def addStat(stats, date, dataType, xcs, via, ipset, https, lang, subdomain,
site):
+key = (date, dataType, xcs, via, ipset, 'https' if https else 'http',
lang, subdomain, site)
+if key in stats:
+stats[key] += 1
+else:
+datetime.strptime(date, '%Y-%m-%d') # Validate date - slow operation,
do it only once per key
+stats[key] = 1
+
+
+columnHdrCache = u'xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
+columnHdrCacheLegacy =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
+columnHdrResult =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,iszero,ison,count'.split(',')
+validSubDomains = {'m', 'zero', 'mobile', 'wap'}
+validHttpCode = {'200', '304'}
+
+
+class WebLogProcessor2(LogProcessor):
+def __init__(self, settingsFile='settings/weblogs2.json'):
+super(WebLogProcessor2, self).__init__(settingsFile, 'web2')
+
+self._configs = None
+self.dateDirRe = re.compile(r'^date=(\d\d\d\d-\d\d-\d\d)$')
+self.fileRe = re.compile(r'^\d+')
+self.combinedFile = os.path.join(self.pathGraphs, 'combined-all.tsv')
+if self.settings.pathCacheLegacy:
+self.pathCacheLegacy =
self.normalizePath(self.settings.pathCacheLegacy)
+else:
+self.pathCacheLegacy = self.settings.pathCacheLegacy
+
+self.legacyFileRe =
re.compile(r'^(zero\.tsv\.log-(\d+)\.gz)__\d+\.tsv$', re.IGNORECASE)
+
+def defaultSettings(self, suffix):
+s = super(WebLogProcessor2, self).defaultSettings(suffix)
+s.pathCacheLegacy = False
+return s
+
+def downloadConfigs(self):
+if self._configs:
+return self._configs
+wiki = self.getWiki()
+#
https://zero.wikimedia.org/w/api.php?action=zeroportaltype=analyticsconfigformat=jsonfm
+configs = wiki('zeroportal', type='analyticsconfig').zeroportal
+for cfs in configs.values():
+for c in cfs:
+c['from'] = datetime.strptime(c['from'], '%Y-%m-%dT%H:%M:%SZ')
+if c.before is None:
+c.before = datetime.max
+else:
+c.before = datetime.strptime(c.before,
'%Y-%m-%dT%H:%M:%SZ')
+c.languages = True if True == c.languages else set(c.languages)
+c.sites = True if True == c.sites else set(c.sites)
+c.via = set(c.via)
+c.ipsets = set(c.ipsets)
+self._configs = configs
+return self._configs
+
+def combineStatsLegacy(self):
+if not self.pathCacheLegacy:
+return {}
+safePrint('Combine legacy stat files')
+# Logs did not contain the VIA X-Analytics tag before this date
+ignoreViaBefore = datetime(2014, 3, 22)
+configs = self.downloadConfigs()
+stats = collections.defaultdict(int)
+for f in os.listdir(self.pathCacheLegacy):
+if not self.legacyFileRe.match(f):
+continue
+for vals in readData(os.path.join(self.pathCacheLegacy, f),
columnHdrCacheLegacy):
+# 0 12 3 4 56 78
9
+# 2014-07-25 DATA 250-99 DIRECT default http ru zero
wikipedia 1000
+if len(vals) != 10:
+if len(vals) == 11 and vals[3] == '':
+safePrint('Fixing extra empty xcs in file %s' % f)
+del vals[3]
+
[MediaWiki-commits] [Gerrit] shell bug, weblogs2 to combine multi-sourced data - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: shell bug, weblogs2 to combine multi-sourced data
..
shell bug, weblogs2 to combine multi-sourced data
Change-Id: Idc09a1ee5434a8e134530cbb722f25430efff6b8
---
M scripts/run-hivezero.sh
A scripts/weblogs2.py
M scripts/zero-counts.hql
3 files changed, 301 insertions(+), 29 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh
index c1a8e96..e04685f 100755
--- a/scripts/run-hivezero.sh
+++ b/scripts/run-hivezero.sh
@@ -8,6 +8,6 @@
for ((day = $3; day = $last; day++)); do
printf -v p %04d-%02d-%02d $1 $2 $day
- echo hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$3 -d
date=$p
- hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$3 -d
date=$p
+ echo hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$day
-d date=$p
+ hive -f zero-counts.hql -d year=$1 -d month=$2 -d day=$day -d
date=$p
done
diff --git a/scripts/weblogs2.py b/scripts/weblogs2.py
new file mode 100644
index 000..2432422
--- /dev/null
+++ b/scripts/weblogs2.py
@@ -0,0 +1,299 @@
+# coding=utf-8
+import StringIO
+import re
+import collections
+from pandas import read_table, pivot_table
+from pandas.core.frame import DataFrame, Series
+import numpy as np
+
+from logprocessor import *
+
+
+def addStat(stats, date, dataType, xcs, via, ipset, https, lang, subdomain,
site):
+key = (date, dataType, xcs, via, ipset, 'https' if https else 'http',
lang, subdomain, site)
+if key in stats:
+stats[key] += 1
+else:
+datetime.strptime(date, '%Y-%m-%d') # Validate date - slow operation,
do it only once per key
+stats[key] = 1
+
+
+columnHdrCache = u'xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
+columnHdrCacheLegacy =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
+columnHdrResult =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,iszero,ison,count'.split(',')
+validSubDomains = {'m', 'zero', 'mobile', 'wap'}
+validHttpCode = {'200', '304'}
+
+
+class WebLogProcessor2(LogProcessor):
+def __init__(self, settingsFile='settings/weblogs2.json'):
+super(WebLogProcessor2, self).__init__(settingsFile, 'web2')
+
+self._configs = None
+self.dateDirRe = re.compile(r'^date=(\d\d\d\d-\d\d-\d\d)$')
+self.fileRe = re.compile(r'^\d+')
+self.combinedFile = os.path.join(self.pathGraphs, 'combined-all.tsv')
+if self.settings.pathCacheLegacy:
+self.pathCacheLegacy =
self.normalizePath(self.settings.pathCacheLegacy)
+else:
+self.pathCacheLegacy = self.settings.pathCacheLegacy
+
+self.legacyFileRe =
re.compile(r'^(zero\.tsv\.log-(\d+)\.gz)__\d+\.tsv$', re.IGNORECASE)
+
+def defaultSettings(self, suffix):
+s = super(WebLogProcessor2, self).defaultSettings(suffix)
+s.pathCacheLegacy = False
+return s
+
+def downloadConfigs(self):
+if self._configs:
+return self._configs
+wiki = self.getWiki()
+#
https://zero.wikimedia.org/w/api.php?action=zeroportaltype=analyticsconfigformat=jsonfm
+configs = wiki('zeroportal', type='analyticsconfig').zeroportal
+for cfs in configs.values():
+for c in cfs:
+c['from'] = datetime.strptime(c['from'], '%Y-%m-%dT%H:%M:%SZ')
+if c.before is None:
+c.before = datetime.max
+else:
+c.before = datetime.strptime(c.before,
'%Y-%m-%dT%H:%M:%SZ')
+c.languages = True if True == c.languages else set(c.languages)
+c.sites = True if True == c.sites else set(c.sites)
+c.via = set(c.via)
+c.ipsets = set(c.ipsets)
+self._configs = configs
+return self._configs
+
+def combineStatsLegacy(self):
+if not self.pathCacheLegacy:
+return {}
+safePrint('Combine legacy stat files')
+# Logs did not contain the VIA X-Analytics tag before this date
+ignoreViaBefore = datetime(2014, 3, 22)
+configs = self.downloadConfigs()
+stats = collections.defaultdict(int)
+for f in os.listdir(self.pathCacheLegacy):
+if not self.legacyFileRe.match(f):
+continue
+for vals in readData(os.path.join(self.pathCacheLegacy, f),
columnHdrCacheLegacy):
+# 0 12 3 4 56 78
9
+# 2014-07-25 DATA 250-99 DIRECT default http ru zero
wikipedia 1000
+if len(vals) != 10:
+if len(vals) == 11 and vals[3] == '':
+safePrint('Fixing extra empty xcs in file %s' % f)
+del vals[3]
+else:
+raise
[MediaWiki-commits] [Gerrit] Zero: unified everything, no opera for 293-41 - change (operations/puppet)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/167037
Change subject: Zero: unified everything, no opera for 293-41
..
Zero: unified everything, no opera for 293-41
* All traffic should now processed as unified
* Cache fragmentation begone!
Change-Id: I5a95fbda0cfb1c4cf1463538c1a65cc1a6211ee1
---
M templates/varnish/zero.inc.vcl.erb
1 file changed, 110 insertions(+), 124 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/37/167037/1
diff --git a/templates/varnish/zero.inc.vcl.erb
b/templates/varnish/zero.inc.vcl.erb
index ae7a709..c3586d3 100644
--- a/templates/varnish/zero.inc.vcl.erb
+++ b/templates/varnish/zero.inc.vcl.erb
@@ -74,130 +74,11 @@
// Short of having netmapper (or equivalent) return a set of match
parameters, I can't see any
// good way to make the rest of this cleaner really.
if (req.http.X-CS2) {
-if (req.http.X-CS2 == 413-02) {
-if (!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) {
-if (req.http.host ~ ^((en|si|simple|ta)\.)?zero\.) {
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 == 470-03) {
-if (!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) {
-if (req.http.host ~ ^((en|bn)\.)?(zero|m)\.) {
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 == 624-02) {
-if (!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) {
-if (req.http.host ~
^((fr|ha|ln|yo|eo|ar|zh|en|es|de)\.)?m\.) {
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 == 605-01) {
-if (!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) {
-if (req.http.host ~ ^((fr|ru|ja|zh|it|de|en|es|ar)\.)?m\.) {
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 == 641-14) {
-if (!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) {
-if (req.http.host ~
^((en|zh|ar|hi|fr|sw|rw|de|es|ko)\.)?m\.) {
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 == 420-01) {
-if (!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) {
-if (req.http.host ~ ^((ar|tl|en|bn|ur)\.)?(zero|m)\.) {
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 == 410-01) {
-if ((!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) ||
req.http.X-Forwarded-By == Opera) {
-if (req.http.host ~ ^((en|ur)\.)?(zero|m)\.) {
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 == 510-11) {
-if (!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) {
-if (req.http.host ~ ^((id|en|zh|ar|hi|ms|jv|su)\.)?zero\.) {
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 == 646-02) {
-if (!req.http.X-Forwarded-By) {
-if (req.http.X-Subdomain == M) {
-set req.http.X-ZeroTLS = 1;
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 ~ ^(456-02|502-13)$) {
-if (!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) {
-if (req.http.X-Subdomain == ZERO) {
-// ZERO only, direct HTTP only
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 ~ ^(614-04|630-86|639-07)$) {
-if (!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) {
-if (req.http.X-Subdomain == M) {
-// M only, direct HTTP only
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 ~ ^(612-03|652-02)$) {
-if ((!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) ||
req.http.X-Forwarded-By == Opera) {
-if (req.http.X-Subdomain == M) {
-// M only, direct HTTP or Opera
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-} else if (req.http.X-CS2 ~ ^(520-18)$) {
-if ((!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) ||
req.http.X-Forwarded-By == Opera) {
-if (req.http.X-Subdomain == ZERO) {
-// ZERO only, direct HTTP or opera
-set req.http.X-CS = req.http.X-CS2;
-}
-}
-}
[MediaWiki-commits] [Gerrit] Updated ZeroBanner to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/166791 Change subject: Updated ZeroBanner to master .. Updated ZeroBanner to master Change-Id: I8a9d56a700c5f458e11e80e7fc682eeab431f3e6 --- M extensions/ZeroBanner 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/91/166791/1 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index 16a19fd..a3febad 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit 16a19fda0625cd38dd9f0297d3313a80f6dfbb0f +Subproject commit a3febadd1b63f92cafd9ef84459fd5b010312e8f -- To view, visit https://gerrit.wikimedia.org/r/166791 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I8a9d56a700c5f458e11e80e7fc682eeab431f3e6 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf2 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated ZeroPortal to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/166792 Change subject: Updated ZeroPortal to master .. Updated ZeroPortal to master Change-Id: I65aee1023958399c6b22770a9b81c832413a31dd --- M extensions/ZeroPortal 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/92/166792/1 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 53ea1ae..3215c45 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 53ea1aeac918f1486663c12b7e1d570eefa75d9f +Subproject commit 3215c4598eb0f6ee21e57cb1c97c4e986a277c26 -- To view, visit https://gerrit.wikimedia.org/r/166792 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I65aee1023958399c6b22770a9b81c832413a31dd Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf2 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated ZeroPortal to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/166794 Change subject: Updated ZeroPortal to master .. Updated ZeroPortal to master Change-Id: I3d6acbc0d38f4731e6f091de103e5eba31650bdc --- M extensions/ZeroPortal 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/94/166794/1 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 1aca020..3215c45 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 1aca020eb86d2b4e64f3c99b220c749ee98c493e +Subproject commit 3215c4598eb0f6ee21e57cb1c97c4e986a277c26 -- To view, visit https://gerrit.wikimedia.org/r/166794 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3d6acbc0d38f4731e6f091de103e5eba31650bdc Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf3 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated ZeroBanner to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/166793 Change subject: Updated ZeroBanner to master .. Updated ZeroBanner to master Change-Id: If5792341aae25aad85210d903425b2f28e14f8e9 --- M extensions/ZeroBanner 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/93/166793/1 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index b50f53a..a3febad 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit b50f53a436e88cfe0dd73b603fc334517f223d55 +Subproject commit a3febadd1b63f92cafd9ef84459fd5b010312e8f -- To view, visit https://gerrit.wikimedia.org/r/166793 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: If5792341aae25aad85210d903425b2f28e14f8e9 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf3 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Moved zerowiki to group0 depl - change (operations/mediawiki-config)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/166800 Change subject: Moved zerowiki to group0 depl .. Moved zerowiki to group0 depl Change-Id: Ib4f1e234ed01bee21bfb69b86b98a1bf588baee2 --- M group0.dblist 1 file changed, 1 insertion(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config refs/changes/00/166800/1 diff --git a/group0.dblist b/group0.dblist index 82d5702..76db358 100644 --- a/group0.dblist +++ b/group0.dblist @@ -2,3 +2,4 @@ test2wiki testwiki testwikidatawiki +zerowiki -- To view, visit https://gerrit.wikimedia.org/r/166800 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ib4f1e234ed01bee21bfb69b86b98a1bf588baee2 Gerrit-PatchSet: 1 Gerrit-Project: operations/mediawiki-config Gerrit-Branch: master Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Disabled ZeroBanner img font - change (operations/mediawiki-config)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/166865 Change subject: Disabled ZeroBanner img font .. Disabled ZeroBanner img font Change-Id: I433cc0f2b198c1c8b7ccf2c3ce6716a2478cc839 --- M wmf-config/mobile.php 1 file changed, 2 insertions(+), 2 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config refs/changes/65/166865/1 diff --git a/wmf-config/mobile.php b/wmf-config/mobile.php index 7cb5681..1bdd562 100644 --- a/wmf-config/mobile.php +++ b/wmf-config/mobile.php @@ -44,8 +44,8 @@ // @TODO: which group(s) on all wikies should Zero allow to flush cache? $wgGroupPermissions['sysop']['jsonconfig-flush'] = true; - $wgZeroBannerFont = '/usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf'; - $wgZeroBannerFontSize = '10'; + // $wgZeroBannerFont = '/usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf'; + // $wgZeroBannerFontSize = '10'; } // Enable loading of desktop-specific resources from MobileFrontend -- To view, visit https://gerrit.wikimedia.org/r/166865 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I433cc0f2b198c1c8b7ccf2c3ce6716a2478cc839 Gerrit-PatchSet: 1 Gerrit-Project: operations/mediawiki-config Gerrit-Branch: master Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Disable ZeroPortal ext on metawiki - change (operations/mediawiki-config)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/166886
Change subject: Disable ZeroPortal ext on metawiki
..
Disable ZeroPortal ext on metawiki
ZeroPortal is no longer hosted on meta, and needs to be deleted.
One possible issue: there were a number of Zero: namespace pages
that used a custom content handler. All these pages have already
been deleted, but there have been some issues with the core
unable to handle missing content models.
Change-Id: I4ad5f91689bce66e33778b34346db98a7e1a6486
---
M wmf-config/CommonSettings.php
M wmf-config/InitialiseSettings.php
2 files changed, 28 insertions(+), 42 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config
refs/changes/86/166886/1
diff --git a/wmf-config/CommonSettings.php b/wmf-config/CommonSettings.php
index 63dadf7..c5130af 100644
--- a/wmf-config/CommonSettings.php
+++ b/wmf-config/CommonSettings.php
@@ -2750,55 +2750,42 @@
require_once( $IP/extensions/ZeroBanner/ZeroBanner.php );
require_once( $IP/extensions/ZeroPortal/ZeroPortal.php );
- // Until meta handles ZeroPortal, need to keep it intact
- if ( $wgDBname == 'zerowiki' ) {
- // zerowiki treats all logged-in users the same as anonymous,
without giving them any extra rights
- // Only sysops and scripts get additional rights on zerowiki
- $zpUserRights = $wgGroupPermissions['user'];
+ // zerowiki treats all logged-in users the same as anonymous, without
giving them any extra rights
+ // Only sysops and scripts get additional rights on zerowiki
+ $zpUserRights = $wgGroupPermissions['user'];
- $wgGroupPermissions['*']['createtalk'] = false;
- $wgGroupPermissions['*']['createpage'] = false;
- $wgGroupPermissions['*']['writeapi'] = false;
- $wgGroupPermissions['user'] = $wgGroupPermissions['*'];
+ $wgGroupPermissions['*']['createtalk'] = false;
+ $wgGroupPermissions['*']['createpage'] = false;
+ $wgGroupPermissions['*']['writeapi'] = false;
+ $wgGroupPermissions['user'] = $wgGroupPermissions['*'];
- // fixme: this should go into groupOverrides or
groupOverrides2, with or without a '+'
- // 'sysop' = array( 'zero-edit', 'zero-script',
'zero-script-ips', 'jsonconfig-flush' ),
- // 'zeroscript' = array( 'zero-script', 'jsonconfig-flush' ),
- // 'zeroscriptips' = array( 'zero-script-ips',
'jsonconfig-flush' ),
+ // fixme: this should go into groupOverrides or groupOverrides2, with
or without a '+'
+ // 'sysop' = array( 'zero-edit', 'zero-script', 'zero-script-ips',
'jsonconfig-flush' ),
+ // 'zeroscript' = array( 'zero-script', 'jsonconfig-flush' ),
+ // 'zeroscriptips' = array( 'zero-script-ips', 'jsonconfig-flush' ),
- $wgGroupPermissions['sysop']['zero-edit'] = true;
- $wgGroupPermissions['sysop']['zero-script'] = true;
- $wgGroupPermissions['sysop']['zero-script-ips'] = true;
- $wgGroupPermissions['sysop']['jsonconfig-flush'] = true;
- $wgGroupPermissions['sysop'] = $wgGroupPermissions['sysop'] +
$zpUserRights;
+ $wgGroupPermissions['sysop']['zero-edit'] = true;
+ $wgGroupPermissions['sysop']['zero-script'] = true;
+ $wgGroupPermissions['sysop']['zero-script-ips'] = true;
+ $wgGroupPermissions['sysop']['jsonconfig-flush'] = true;
+ $wgGroupPermissions['sysop'] = $wgGroupPermissions['sysop'] +
$zpUserRights;
- $wgGroupPermissions['zeroscript']['zero-script'] = true;
- $wgGroupPermissions['zeroscript']['jsonconfig-flush'] = true;
- $wgGroupPermissions['zeroscript'] =
$wgGroupPermissions['zeroscript'] + $zpUserRights;
+ $wgGroupPermissions['zeroscript']['zero-script'] = true;
+ $wgGroupPermissions['zeroscript']['jsonconfig-flush'] = true;
+ $wgGroupPermissions['zeroscript'] = $wgGroupPermissions['zeroscript'] +
$zpUserRights;
- $wgGroupPermissions['zeroscriptips']['zero-script-ips'] = true;
- $wgGroupPermissions['zeroscriptips']['jsonconfig-flush'] = true;
- $wgGroupPermissions['zeroscriptips'] =
$wgGroupPermissions['zeroscriptips'] + $zpUserRights;
+ $wgGroupPermissions['zeroscriptips']['zero-script-ips'] = true;
+ $wgGroupPermissions['zeroscriptips']['jsonconfig-flush'] = true;
+ $wgGroupPermissions['zeroscriptips'] =
$wgGroupPermissions['zeroscriptips'] + $zpUserRights;
- $wgZeroPortalImpersonateUser = 'Impersonator';
+ $wgZeroPortalImpersonateUser = 'Impersonator';
- unset( $zpUserRights );
+ unset( $zpUserRights );
- $wgUsersNotifiedOnAllChanges[] = 'ABaso(WMF)';
- $wgUsersNotifiedOnAllChanges[] = 'Dfoy';
-
[MediaWiki-commits] [Gerrit] Zero: Updated 410-01 to support Opera https - change (operations/puppet)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/166936
Change subject: Zero: Updated 410-01 to support Opera https
..
Zero: Updated 410-01 to support Opera https
Change-Id: I53ee6e8d81f42b662e2743913df837af1ca34f5a
---
M templates/varnish/zero.inc.vcl.erb
1 file changed, 1 insertion(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/36/166936/1
diff --git a/templates/varnish/zero.inc.vcl.erb
b/templates/varnish/zero.inc.vcl.erb
index e4196a8..ae7a709 100644
--- a/templates/varnish/zero.inc.vcl.erb
+++ b/templates/varnish/zero.inc.vcl.erb
@@ -111,7 +111,7 @@
}
}
} else if (req.http.X-CS2 == 410-01) {
-if (!req.http.X-Forwarded-Proto (!req.http.X-Forwarded-By ||
req.http.X-Forwarded-By == Opera)) {
+if ((!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) ||
req.http.X-Forwarded-By == Opera) {
if (req.http.host ~ ^((en|ur)\.)?(zero|m)\.) {
set req.http.X-CS = req.http.X-CS2;
}
--
To view, visit https://gerrit.wikimedia.org/r/166936
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I53ee6e8d81f42b662e2743913df837af1ca34f5a
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fixed file perm on Gemfile per bug 71932 - change (mediawiki...ZeroBanner)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/166352 Change subject: Fixed file perm on Gemfile per bug 71932 .. Fixed file perm on Gemfile per bug 71932 Change-Id: Ifa02dbf9b0f0c05464ea16e9de6093b3606e748f --- M tests/browser/Gemfile 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroBanner refs/changes/52/166352/1 diff --git a/tests/browser/Gemfile b/tests/browser/Gemfile old mode 100755 new mode 100644 -- To view, visit https://gerrit.wikimedia.org/r/166352 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ifa02dbf9b0f0c05464ea16e9de6093b3606e748f Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/ZeroBanner Gerrit-Branch: master Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Generate JSON font info file - change (mediawiki...UniversalLanguageSelector)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/165698
Change subject: Generate JSON font info file
..
Generate JSON font info file
This patch generates an additional ext.uls.webfonts.repository.json
file, which is the pure data version of the ext.uls.webfonts.repository.js
This file is used by the backend-based systems such as ZeroBanner.
Change-Id: I7fd4031c505909ab5a996561d4857bab2033e028
---
M data/fontrepo/scripts/compile.php
A resources/ext.uls.webfonts.repository.json
2 files changed, 2 insertions(+), 0 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/UniversalLanguageSelector
refs/changes/98/165698/1
diff --git a/data/fontrepo/scripts/compile.php
b/data/fontrepo/scripts/compile.php
index 1c6986c..f751008 100644
--- a/data/fontrepo/scripts/compile.php
+++ b/data/fontrepo/scripts/compile.php
@@ -95,5 +95,6 @@
JAVASCRIPT;
file_put_contents( '../../../resources/js/ext.uls.webfonts.repository.js', $js
);
+file_put_contents( '../../../resources/ext.uls.webfonts.repository.json',
$json );
echo Done.\n;
diff --git a/resources/ext.uls.webfonts.repository.json
b/resources/ext.uls.webfonts.repository.json
new file mode 100644
index 000..6eb700c
--- /dev/null
+++ b/resources/ext.uls.webfonts.repository.json
@@ -0,0 +1 @@
+{base:..\/data\/fontrepo\/fonts\/,languages:{adx:[Jomolhari],af:[system,ComicNeue,OpenDyslexic],ahr:[Lohit
Marathi],akk:[Akkadian],am:[AbyssinicaSIL],ang:[system,Junicode],ar:[system,Amiri],arb:[system,Amiri],arc:[Estrangelo
Edessa,East Syriac Adiabene,SertoUrhoy],as:[system,Lohit
Assamese],bbc:[system,Pangururan],bh:[Lohit
Devanagari],bho:[Lohit
Devanagari],bk:[system,ComicNeue,OpenDyslexic],bn:[Siyam
Rupali,Lohit Bengali],bo:[Jomolhari],bod:[Jomolhari],bpy:[Siyam
Rupali,Lohit
Bengali],btk:[system,Pangururan],bug:[Saweri],ca:[system,ComicNeue,OpenDyslexic],cdo:[system,CharisSIL],ckb:[system,Lateef,Scheherazade,Amiri],cr:[OskiEast],cy:[system,ComicNeue,OpenDyslexic],da:[system,ComicNeue,OpenDyslexic],de:[system,ComicNeue,OpenDyslexic],dre:[Jomolhari],dv:[FreeFont-Thaana],dz:[Jomolhari],en:[system,ComicNeue,OpenDyslexic],es:[system,ComicNeue,OpenDyslexic],et:[system,ComicNeue,OpenDyslexic],fa:[system,Iranian
Sans,Iranian
Serif,Lateef,Nazli,Scheherazade],fi:[system,ComicNeue,OpenDyslexic],fo:[system,ComicNeue,OpenDyslexic],fr:[system,ComicNeue,OpenDyslexic],fy:[system,ComicNeue,OpenDyslexic],ga:[system,ComicNeue,OpenDyslexic],gd:[system,ComicNeue,OpenDyslexic],gez:[AbyssinicaSIL],gl:[system,ComicNeue,OpenDyslexic],goe:[Jomolhari],gom:[Lohit
Devanagari],grc:[system,GentiumPlus],gu:[Lohit
Gujarati],hbo:[Taamey Frank CLM,Alef],he:[system,Alef,Miriam
CLM,Taamey Frank CLM],hi:[Lohit
Devanagari],hu:[system,ComicNeue,OpenDyslexic],hut:[Jomolhari],id:[system,ComicNeue,OpenDyslexic],ii:[Nuosu
SIL],is:[system,ComicNeue,OpenDyslexic],it:[system,ComicNeue,OpenDyslexic],iu:[system,OskiEast],jv:[system,Tuladha
Jejeg],jv-java:[Tuladha
Jejeg],kbg:[Jomolhari],khg:[Jomolhari],km:[KhmerOSbattambang,Hanuman,KhmerOS,Nokora
Regular,Suwannaphum],kn:[Lohit Kannada,Gubbi],kok:[Lohit
Devanagari],kte:[Jomolhari],lb:[system,ComicNeue,OpenDyslexic],lbj:[Jomolhari],lhm:[Jomolhari],li:[system,ComicNeue,OpenDyslexic],lo:[Phetsarath],loy:[Jomolhari],luk:[Jomolhari],lya:[Jomolhari],mai:[Lohit
Devanagari],mak:[Saweri],mi:[system,ComicNeue,OpenDyslexic],ml:[system,AnjaliOldLipi,Meera],mr:[Lohit
Marathi],ms:[system,ComicNeue,OpenDyslexic],muk:[Jomolhari],mul:[system,Autonym],my:[TharLon,Myanmar3,Padauk],nan:[system,CharisSIL,Doulos
SIL],nb:[system,ComicNeue,OpenDyslexic],ne:[Lohit
Nepali,Madan],nl:[system,ComicNeue,OpenDyslexic],oc:[system,ComicNeue,OpenDyslexic],ola:[Jomolhari],or:[Lohit
Odia,Utkal],otb:[Jomolhari],pa:[Lohit
Punjabi,Saab],pal:[Shapour],peo:[Artaxerxes],pl:[system,ComicNeue,OpenDyslexic],pt:[system,ComicNeue,OpenDyslexic],sa:[Lohit
Devanagari],saz:[Pagul],si:[system,lklug],sq:[system,ComicNeue,OpenDyslexic],sux:[Akkadian],sv:[system,ComicNeue,OpenDyslexic],sw:[system,ComicNeue,OpenDyslexic],syc:[Estrangelo
Edessa,East Syriac Adiabene,SertoUrhoy],ta:[system,Lohit
Tamil,Lohit Tamil
Classical,Thendral,Thenee],tcn:[Jomolhari],tcy:[Lohit
Kannada,Gubbi],te:[Lohit
Telugu],thw:[Jomolhari],ti:[AbyssinicaSIL],tig:[AbyssinicaSIL],tl:[system,ComicNeue,OpenDyslexic],tr:[system,ComicNeue,OpenDyslexic],tsk:[Jomolhari],ur:[system,Hussaini
Nastaleeq,NafeesWeb],wa:[system,ComicNeue,OpenDyslexic],xct:[Jomolhari],yi:[system,Alef],zau:[Jomolhari]},fonts:{AbyssinicaSIL:{version:1.500,eot:AbyssinicaSIL\/AbyssinicaSIL-R.eot,ttf:AbyssinicaSIL\/AbyssinicaSIL-R.ttf,woff:AbyssinicaSIL\/AbyssinicaSIL-R.woff},Akkadian:{version:2.56,eot:Akkadian\/Akkadian.eot,ttf:Akkadian\/Akkadian.ttf,woff:Akkadian\/Akkadian.woff},Alef:{version:1.0,ttf:Alef\/Alef-Regular.ttf,eot:Alef\/Alef-Regular.eot,woff:Alef\/Alef-Regular.woff,variants:{bold:Alef
Bold}},Alef
[MediaWiki-commits] [Gerrit] Downloaded open source font from http://unifoundry.com/unifo... - change (mediawiki...ZeroBanner)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/165699 Change subject: Downloaded open source font from http://unifoundry.com/unifont.html .. Downloaded open source font from http://unifoundry.com/unifont.html Change-Id: I75a62cb6a663c869b52aba1cf30761d8460be708 --- M ZeroBanner.php A unifont/LICENSE.txt A unifont/README A unifont/unifont_csur-7.0.03.ttf 4 files changed, 374 insertions(+), 2 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroBanner refs/changes/99/165699/1 diff --git a/ZeroBanner.php b/ZeroBanner.php index f8e34db..51774d1 100644 --- a/ZeroBanner.php +++ b/ZeroBanner.php @@ -119,7 +119,7 @@ $wgZeroBannerClusterDomain = 'org'; /** @var false|string $wgZeroBannerFontSize size of the font used for the image. When given, ignores *ImageSize vals */ -$wgZeroBannerFontSize = false; +$wgZeroBannerFontSize = '10'; /** @var string $wgZeroBannerImageSize size of the image banner. A 3px border will be added */ $wgZeroBannerImageSize = '174x'; @@ -128,7 +128,7 @@ $wgZeroBannerErrImageSize = '194x'; /** @var string|false $wgZeroBannerFont use a specific font for image banners */ -$wgZeroBannerFont = false; // 'FreeSans' ? +$wgZeroBannerFont = __DIR__ . '/unifont/unifont_csur-7.0.03.ttf'; $hook = 'ZeroBanner\\PageRenderingHooks::'; $wgHooks['BeforePageDisplayMobile'][] = $hook . 'onBeforePageDisplay'; diff --git a/unifont/LICENSE.txt b/unifont/LICENSE.txt new file mode 100644 index 000..6b540a8 --- /dev/null +++ b/unifont/LICENSE.txt @@ -0,0 +1,371 @@ +LICENSE +--- +The source code for everything except the compiled fonts in this current +release is licensed as follows: + + License for this current distribution of program source + files (i.e., everything except the fonts) is released under + the terms of the GNU General Public License version 2, + or (at your option) a later version. + + See the section below for a copy of the GNU General Public License + version 2. + +The license for the compiled fonts is covered by the above GPL terms +with the GNU font embedding exception, as follows: + + As a special exception, if you create a document which uses this font, + and embed this font or unaltered portions of this font into the document, + this font does not by itself cause the resulting document to be covered + by the GNU General Public License. This exception does not however + invalidate any other reasons why the document might be covered by the + GNU General Public License. If you modify this font, you may extend + this exception to your version of the font, but you are not obligated + to do so. If you do not wish to do so, delete this exception statement + from your version. + +See http://www.gnu.org/licenses/gpl-faq.html#FontException; for more details. + + +GPL VERSION 2 +- + +GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you
[MediaWiki-commits] [Gerrit] Minor doc fix for wfShellExecWithStderr() - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/165525 Change subject: Minor doc fix for wfShellExecWithStderr() .. Minor doc fix for wfShellExecWithStderr() Change-Id: I9d5fe442c6e85d61c1cdec8c42caab06b37856b1 --- M includes/GlobalFunctions.php 1 file changed, 3 insertions(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/25/165525/1 diff --git a/includes/GlobalFunctions.php b/includes/GlobalFunctions.php index 03bdd76..2c26fef 100644 --- a/includes/GlobalFunctions.php +++ b/includes/GlobalFunctions.php @@ -2986,7 +2986,9 @@ * function, as all the arguments to wfShellExec can become unwieldy. * * @note This also includes errors from limit.sh, e.g. if $wgMaxShellFileSize is exceeded. - * @param string $cmd Command line, properly escaped for shell. + * @param string|string[] $cmd If string, a properly shell-escaped command line, + * or an array of unescaped arguments, in which case each value will be escaped + * Example: [ 'convert', '-font', 'font name' ] would produce 'convert' '-font' 'font name' * @param null|mixed $retval Optional, will receive the program's exit code. * (non-zero is usually failure) * @param array $environ Optional environment variables which should be -- To view, visit https://gerrit.wikimedia.org/r/165525 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I9d5fe442c6e85d61c1cdec8c42caab06b37856b1 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: master Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Remove duplicate param escaping code - change (mediawiki/core)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/165679
Change subject: Remove duplicate param escaping code
..
Remove duplicate param escaping code
wfEscapeShellArg() can handle multiple params, escaping each.
This patch changes wfShellExec() to call wfEscapeShellArg() directly
instead of doing the gluing itself.
Change-Id: I7a0761cc2ba98c210a9eacadd12da407d933e42a
---
M includes/GlobalFunctions.php
1 file changed, 1 insertion(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/79/165679/1
diff --git a/includes/GlobalFunctions.php b/includes/GlobalFunctions.php
index 2c26fef..1aa936c 100644
--- a/includes/GlobalFunctions.php
+++ b/includes/GlobalFunctions.php
@@ -2796,11 +2796,7 @@
}
if ( is_array( $cmd ) ) {
// Command line may be given as an array, escape each value and
glue them together with a space
- $cmdVals = array();
- foreach ( $cmd as $val ) {
- $cmdVals[] = wfEscapeShellArg( $val );
- }
- $cmd = implode( ' ', $cmdVals );
+ $cmd = call_user_func_array( 'wfEscapeShellArg', $cmd );
}
$cmd = $envcmd . $cmd;
--
To view, visit https://gerrit.wikimedia.org/r/165679
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7a0761cc2ba98c210a9eacadd12da407d933e42a
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Enable graph ext on outreachwiki - change (operations/mediawiki-config)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/165099 Change subject: Enable graph ext on outreachwiki .. Enable graph ext on outreachwiki Change-Id: I802a171e0cbc5a1ea43566cbe794ee9da6f31175 --- M wmf-config/InitialiseSettings.php 1 file changed, 1 insertion(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config refs/changes/99/165099/1 diff --git a/wmf-config/InitialiseSettings.php b/wmf-config/InitialiseSettings.php index 076a10d..34d8dbe 100644 --- a/wmf-config/InitialiseSettings.php +++ b/wmf-config/InitialiseSettings.php @@ -11946,6 +11946,7 @@ 'labswiki' = true, 'mediawikiwiki' = true, 'metawiki' = true, + 'outreachwiki' = true, 'zerowiki' = true, ), -- To view, visit https://gerrit.wikimedia.org/r/165099 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I802a171e0cbc5a1ea43566cbe794ee9da6f31175 Gerrit-PatchSet: 1 Gerrit-Project: operations/mediawiki-config Gerrit-Branch: master Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Added 4 ppl to notifyOnAllChanges for zero portal - change (operations/mediawiki-config)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/164902
Change subject: Added 4 ppl to notifyOnAllChanges for zero portal
..
Added 4 ppl to notifyOnAllChanges for zero portal
Change-Id: I4fc0f8f7de153b5ce9f3e4f0a4f484c70e69beb5
---
M wmf-config/CommonSettings.php
1 file changed, 5 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config
refs/changes/02/164902/1
diff --git a/wmf-config/CommonSettings.php b/wmf-config/CommonSettings.php
index 36d6aa6..85b05c0 100644
--- a/wmf-config/CommonSettings.php
+++ b/wmf-config/CommonSettings.php
@@ -2779,6 +2779,11 @@
$wgZeroPortalImpersonateUser = 'Impersonator';
unset( $zpUserRights );
+
+ $wgUsersNotifiedOnAllChanges[] = 'ABaso(WMF)';
+ $wgUsersNotifiedOnAllChanges[] = 'Dfoy';
+ $wgUsersNotifiedOnAllChanges[] = 'Jhobs';
+ $wgUsersNotifiedOnAllChanges[] = 'Yurik';
} else {
// metawiki
$wgGroupPermissions['zeroadmin']['zero-edit'] = true;
--
To view, visit https://gerrit.wikimedia.org/r/164902
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I4fc0f8f7de153b5ce9f3e4f0a4f484c70e69beb5
Gerrit-PatchSet: 1
Gerrit-Project: operations/mediawiki-config
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Removed isZeroSite()'s caching - change (mediawiki...ZeroBanner)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/164519
Change subject: Removed isZeroSite()'s caching
..
Removed isZeroSite()'s caching
Change-Id: I1d1611d1f959bfd33cce8e5d42affd5a634faf77
---
M includes/PageRendering.php
1 file changed, 3 insertions(+), 7 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroBanner
refs/changes/19/164519/1
diff --git a/includes/PageRendering.php b/includes/PageRendering.php
index 88bcb95..1dd2645 100644
--- a/includes/PageRendering.php
+++ b/includes/PageRendering.php
@@ -651,13 +651,9 @@
* @return bool
*/
private function isZeroSite() {
- static $isZero = null;
- if ( $isZero === null ) {
- $info = $this-getWikiInfo();
- // @FIXME: this check should only see if this is a
mobileview, the rest is a check in partner's config
- $isZero = $info[2] === 'wikipedia'
MobileContext::singleton()-shouldDisplayMobileView();
- }
- return $isZero;
+ $info = $this-getWikiInfo();
+ // @FIXME: this check should only see if this is a mobileview,
the rest is a check in partner's config
+ return $info[2] === 'wikipedia'
MobileContext::singleton()-shouldDisplayMobileView();
}
/**
--
To view, visit https://gerrit.wikimedia.org/r/164519
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1d1611d1f959bfd33cce8e5d42affd5a634faf77
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/ZeroBanner
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] renamed var, unset it during initialization - change (mediawiki...ZeroPortal)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/164603
Change subject: renamed var, unset it during initialization
..
renamed var, unset it during initialization
Change-Id: Ifc011ad49b462d2db3aa4e3ec822c7b7cfd74cdf
---
M ZeroPortal.php
1 file changed, 7 insertions(+), 6 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroPortal
refs/changes/03/164603/1
diff --git a/ZeroPortal.php b/ZeroPortal.php
index b9249ad..85150f6 100644
--- a/ZeroPortal.php
+++ b/ZeroPortal.php
@@ -31,20 +31,21 @@
'url' = 'https://www.mediawiki.org/wiki/Extension:ZeroPortal',
);
-$cwd = __DIR__ . DIRECTORY_SEPARATOR;
-$wgMessagesDirs['ZeroPortal'] = $cwd . 'i18n';
-$wgExtensionMessagesFiles['ZeroPortal'] = $cwd . 'ZeroPortal.i18n.php';
+$zpDir = __DIR__ . DIRECTORY_SEPARATOR;
+$wgMessagesDirs['ZeroPortal'] = $zpDir . 'i18n';
+$wgExtensionMessagesFiles['ZeroPortal'] = $zpDir . 'ZeroPortal.i18n.php';
// autoload extension classes
-$cwd .= 'includes' . DIRECTORY_SEPARATOR;
+$zpDir .= 'includes' . DIRECTORY_SEPARATOR;
foreach ( array(
'ApiZeroPortal',
'ConfigPageHooks',
- 'ZeroConfigView',
'PortalSpecialPage',
+ 'ZeroConfigView',
) as $key = $class ) {
- $wgAutoloadClasses['ZeroPortal\\' . ( is_string( $key ) ? $key : $class
)] = $cwd . $class . '.php';
+ $wgAutoloadClasses['ZeroPortal\\' . ( is_string( $key ) ? $key : $class
)] = $zpDir . $class . '.php';
}
+unset( $zpDir );
$wgResourceModules['zeroportal.config'] = array(
'dependencies' = 'zerobanner.config.styles', // defined in ZeroBanner
ext, used for sample banners
--
To view, visit https://gerrit.wikimedia.org/r/164603
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifc011ad49b462d2db3aa4e3ec822c7b7cfd74cdf
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/ZeroPortal
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Security and styling fix - change (mediawiki...Graph)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/164708
Change subject: Security and styling fix
..
Security and styling fix
* Per https://bugzilla.wikimedia.org/show_bug.cgi?id=69623#c7
vg.config.safeMode is true unless vg.config.domainWhiteList is false
* mw-wiki-graph css: display: inline-block
Change-Id: I8d880ef762e00506e0e1fb570bedd1be3d64dbac
---
M js/graph.js
M styles/common.less
2 files changed, 4 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Graph
refs/changes/08/164708/1
diff --git a/js/graph.js b/js/graph.js
index 2a00ea8..4bf7f5c 100644
--- a/js/graph.js
+++ b/js/graph.js
@@ -1,9 +1,7 @@
( function( $ ) {
$( function() {
vg.config.domainWhiteList = mw.config.get('wgGraphDataDomains');
- if (vg.config.domainWhiteList) {
- vg.config.safeMode = true;
- }
+ vg.config.safeMode = vg.config.domainWhiteList !== false;
$('.mw-wiki-graph').each(function () {
var definition = $(this).data('spec'),
el = this;
diff --git a/styles/common.less b/styles/common.less
index e69de29..5f32fa3 100644
--- a/styles/common.less
+++ b/styles/common.less
@@ -0,0 +1,3 @@
+.mw-wiki-graph {
+ display: inline-block
+}
--
To view, visit https://gerrit.wikimedia.org/r/164708
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8d880ef762e00506e0e1fb570bedd1be3d64dbac
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Graph
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Per Timo: minor javascript optimizations - change (mediawiki...ZeroBanner)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/164454
Change subject: Per Timo: minor javascript optimizations
..
Per Timo: minor javascript optimizations
* semicolon in the dynamically inserted javascript (zcmd=js-banner)
* attach click event handler to the document instead of each anchor
Change-Id: I8e65195c284a8299ff2d122217c26be5351974c5
---
M includes/ZeroSpecialPage.php
M modules/interstitial.js
2 files changed, 3 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroBanner
refs/changes/54/164454/1
diff --git a/includes/ZeroSpecialPage.php b/includes/ZeroSpecialPage.php
index 6be0f08..0594a23 100644
--- a/includes/ZeroSpecialPage.php
+++ b/includes/ZeroSpecialPage.php
@@ -431,7 +431,7 @@
$bannerHtml =
PageRendering::renderBanner( $state, $config, null, null, $isFilePage );
$cfg = PageRendering::getJsConfigBlock(
$this, $id, $config, (bool)$bannerHtml );
if ( $bannerHtml ) {
- $banner = 'document.write(' .
Xml::encodeJsVar( $bannerHtml ) . ')';
+ $banner = 'document.write(' .
Xml::encodeJsVar( $bannerHtml ) . ');';
}
} else {
$cfg = PageRendering::getJsConfigBlock(
$this, $id, $config, false );
diff --git a/modules/interstitial.js b/modules/interstitial.js
index e4a6b15..864b589 100644
--- a/modules/interstitial.js
+++ b/modules/interstitial.js
@@ -71,12 +71,12 @@
}
} );
-$( 'a' ).on( 'click', function ( ev ) {
+$( document.body ).on( 'click', 'a', function ( ev ) {
var knownSites = [
// a hardcoded list of our sites - avoids api call if not
listed
'm.wikipedia', 'zero.wikipedia'
],
-$target = $( this ), // onclick a element
+$target = $( ev.target ), // onclick a element
isSpecialCase = false, // true if interstitial should be shown
despite not being a redirect
search = $target.prop( 'search' ), // query part of the link
target URL
isImage = $target.hasClass( 'image' ), // is this a link to an
image page
--
To view, visit https://gerrit.wikimedia.org/r/164454
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8e65195c284a8299ff2d122217c26be5351974c5
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/ZeroBanner
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updating ZeroBanner ZeroPortal to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/164134 Change subject: Updating ZeroBanner ZeroPortal to master .. Updating ZeroBanner ZeroPortal to master Change-Id: Ib8734d0a432873095bdc5a6298eb4b73d1f1a436 --- M extensions/ZeroBanner M extensions/ZeroPortal 2 files changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/34/164134/1 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index a3b661a..99f608f 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit a3b661ae2dd33ffccbe9bb7757278f5351314dcd +Subproject commit 99f608fb244086640b0415ddb5167b5a8ed0296f diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 5d62a7b..b737a48 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 5d62a7b614e39a49426435bb2dce0d51a3e3a7f4 +Subproject commit b737a482cfbd687307adc6f41ee6c5fd728d3c4c -- To view, visit https://gerrit.wikimedia.org/r/164134 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ib8734d0a432873095bdc5a6298eb4b73d1f1a436 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf1 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updating ZeroBanner to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/164136 Change subject: Updating ZeroBanner to master .. Updating ZeroBanner to master Change-Id: I6b7276ccb3390106c0c71e19d2a780edef6e0b74 --- M extensions/ZeroBanner 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/36/164136/1 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index a3b661a..99f608f 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit a3b661ae2dd33ffccbe9bb7757278f5351314dcd +Subproject commit 99f608fb244086640b0415ddb5167b5a8ed0296f -- To view, visit https://gerrit.wikimedia.org/r/164136 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I6b7276ccb3390106c0c71e19d2a780edef6e0b74 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf1 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updating ZeroBanner to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/164137 Change subject: Updating ZeroBanner to master .. Updating ZeroBanner to master Change-Id: Ibfab153e1b6612a7eec357b59b1fab08cde5db16 --- M extensions/ZeroBanner 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/37/164137/1 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index f794a84..99f608f 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit f794a84d5f8979b49f8c714404f3f48e6b4403d0 +Subproject commit 99f608fb244086640b0415ddb5167b5a8ed0296f -- To view, visit https://gerrit.wikimedia.org/r/164137 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ibfab153e1b6612a7eec357b59b1fab08cde5db16 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.24wmf22 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fix private access from closure 5.3 vs 5.4 bug - change (mediawiki...ZeroBanner)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/164145
Change subject: Fix private access from closure 5.3 vs 5.4 bug
..
Fix private access from closure 5.3 vs 5.4 bug
Change-Id: Ia458fa7e2a5677fabeaebc5e43192d0d96e721fa
---
M includes/PageRendering.php
M includes/ZeroConfig.php
2 files changed, 7 insertions(+), 9 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroBanner
refs/changes/45/164145/1
diff --git a/includes/PageRendering.php b/includes/PageRendering.php
index 88bcb95..db1c5e8 100644
--- a/includes/PageRendering.php
+++ b/includes/PageRendering.php
@@ -87,7 +87,7 @@
$cfg['enableHttps'] = $config-enableHttps();
$cfg['sites'] = $config-sites(); // todo: remove this
value once it's out of cache
$cfg['domains'] = $config-domains();
- $cfg['proxy'] = $config-proxy();
+ $cfg['proxy'] = $config-proxy();1
$cfg['ipset'] = $config-ipset();
$cfg['bannerUrl'] = $config-bannerUrl();
$cfg['bannerWarning'] = $config-bannerWarning();
diff --git a/includes/ZeroConfig.php b/includes/ZeroConfig.php
index 6837157..52fa8f1 100644
--- a/includes/ZeroConfig.php
+++ b/includes/ZeroConfig.php
@@ -30,7 +30,9 @@
private $mode = null;
private $ipset = null;
private $proxy = null;
- private $enabledValue = false;
+
+ /** @var bool DO NOT use from outside of the class. Public needed for
the closure call */
+ public $enabledValue = false;
const ignoreNetwork = 0x1;
const ignoreDisabled = 0x2;
@@ -333,12 +335,12 @@
return function ( JCValue $jcv, array $path ) use ( $self ) {
if ( $jcv-isMissing() ) {
$jcv-setValue( true );
- $self-setEnabled( true );
+ $self-enabledValue = true;
return true;
}
$v = $jcv-getValue();
if ( is_bool( $v ) ) {
- $self-setEnabled( $v );
+ $self-enabledValue = $v;
return true;
}
if ( JCUtils::isList( $v ) count( $v ) === 2 ) {
@@ -388,7 +390,7 @@
$until-format(
'Y-m-d\TH:i:s\Z' )
) );
$now = new DateTime( 'now', new
DateTimeZone( 'UTC' ) );
- $self-setEnabled( $now $from
$now $until );
+ $self-enabledValue = $now
$from $now $until;
return true;
}
}
@@ -672,9 +674,5 @@
}
return true;
};
- }
-
- private function setEnabled( $value ) {
- $this-enabledValue = $value;
}
}
--
To view, visit https://gerrit.wikimedia.org/r/164145
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia458fa7e2a5677fabeaebc5e43192d0d96e721fa
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/ZeroBanner
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updating to ZeroBanner master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/164146 Change subject: Updating to ZeroBanner master .. Updating to ZeroBanner master Change-Id: I3ccd091f0d175453cd55e61dd9d669540dd94b4a --- M extensions/ZeroBanner 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/46/164146/1 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index 99f608f..bb8be99 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit 99f608fb244086640b0415ddb5167b5a8ed0296f +Subproject commit bb8be99bc91caa013c5dc2409f62353c22d122bc -- To view, visit https://gerrit.wikimedia.org/r/164146 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3ccd091f0d175453cd55e61dd9d669540dd94b4a Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.24wmf22 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updating to ZeroBanner master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/164147 Change subject: Updating to ZeroBanner master .. Updating to ZeroBanner master Change-Id: Iddc3d42746ba1530387d2b2c83d1dff555336ddc --- M extensions/ZeroBanner 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/47/164147/1 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index 99f608f..bb8be99 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit 99f608fb244086640b0415ddb5167b5a8ed0296f +Subproject commit bb8be99bc91caa013c5dc2409f62353c22d122bc -- To view, visit https://gerrit.wikimedia.org/r/164147 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Iddc3d42746ba1530387d2b2c83d1dff555336ddc Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf1 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Added 'name' value to jsConfigBlock, init cleanup - change (mediawiki...ZeroBanner)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/163804
Change subject: Added 'name' value to jsConfigBlock, init cleanup
..
Added 'name' value to jsConfigBlock, init cleanup
* A localized 'name' will be included in the js config block if available
* ext initialization has been cleaned up a bit
Change-Id: Ic4aaf5883496c53e3dc43c22131700d4cdc533b2
---
M ZeroBanner.php
M includes/PageRendering.php
M includes/ZeroSpecialPage.php
3 files changed, 29 insertions(+), 28 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroBanner
refs/changes/04/163804/1
diff --git a/ZeroBanner.php b/ZeroBanner.php
index 2d06d09..f8e34db 100644
--- a/ZeroBanner.php
+++ b/ZeroBanner.php
@@ -26,35 +26,34 @@
'url' = 'https://www.mediawiki.org/wiki/Extension:ZeroBanner',
);
-$cwd = __DIR__ . DIRECTORY_SEPARATOR;
-$wgMessagesDirs['ZeroBanner'] = $cwd . 'i18n';
-$wgExtensionMessagesFiles['ZeroBanner'] = $cwd . 'ZeroBanner.i18n.php';
-$wgExtensionMessagesFiles['ZeroBannerAlias'] = $cwd . ZeroBanner.alias.php;
+$zbDir = __DIR__ . DIRECTORY_SEPARATOR;
+$wgMessagesDirs['ZeroBanner'] = $zbDir . 'i18n';
+$wgExtensionMessagesFiles['ZeroBanner'] = $zbDir . 'ZeroBanner.i18n.php';
+$wgExtensionMessagesFiles['ZeroBannerAlias'] = $zbDir . ZeroBanner.alias.php;
$wgSpecialPages['ZeroRatedMobileAccess'] = 'ZeroBanner\ZeroSpecialPage';
$wgSpecialPageGroups['ZeroRatedMobileAccess'] = 'other';
-$wgExtensionMessagesFiles['ZeroRatedMobileAccessAlias'] = $cwd .
ZeroBanner.alias.php;
+$wgExtensionMessagesFiles['ZeroRatedMobileAccessAlias'] = $zbDir .
ZeroBanner.alias.php;
// autoload extension classes
-$cwd .= 'includes' . DIRECTORY_SEPARATOR;
-$autoloadClasses = array (
- 'ApiRawJsonPrinter' = 'ApiZeroBanner',
- 'ApiZeroBanner' = 'ApiZeroBanner',
- 'MccMncLogging' = 'MccMncLogging',
- 'PageRendering' = 'PageRendering',
- 'PageRenderingHooks' = 'PageRenderingHooks',
- 'ZeroConfig' = 'ZeroConfig',
- 'ZeroSpecialPage' = 'ZeroSpecialPage',
-);
-foreach ( $autoloadClasses as $className = $classFilename ) {
- $wgAutoloadClasses['ZeroBanner\\' . $className] = $cwd . $classFilename
. '.php';
+$zbDir .= 'includes' . DIRECTORY_SEPARATOR;
+foreach ( array(
+ 'ApiRawJsonPrinter' = 'ApiZeroBanner',
+ 'ApiZeroBanner',
+ 'MccMncLogging',
+ 'PageRendering',
+ 'PageRenderingHooks',
+ 'ZeroConfig',
+ 'ZeroSpecialPage',
+ ) as $key = $class ) {
+ $wgAutoloadClasses['ZeroBanner\\' . ( is_string( $key ) ? $key : $class
)] = $zbDir . $class . '.php';
}
+unset( $zbDir );
-$localBasePath = __DIR__;
$remoteExtPath = 'ZeroBanner';
$wgResourceModules['zerobanner.styles'] = array(
- 'localBasePath' = $localBasePath,
+ 'localBasePath' = __DIR__,
'remoteExtPath' = $remoteExtPath,
'targets' = 'mobile',
'styles' = array( 'modules/banner.css', 'modules/redux.less' ),
@@ -62,7 +61,7 @@
// this resource is used by the ZeroPortal to display banners on the desktop
site
$wgResourceModules['zerobanner.config.styles'] = array(
- 'localBasePath' = $localBasePath,
+ 'localBasePath' = __DIR__,
'remoteExtPath' = $remoteExtPath,
'targets' = array( 'desktop', 'mobile' ),
'styles' = 'modules/banner.css',
@@ -70,9 +69,9 @@
$wgResourceModules['zerobanner'] = array(
'dependencies' = array( 'mobile.stable', 'mobile.templates',
'zerobanner.styles', 'mobile.overlays' ),
- 'localBasePath' = $localBasePath,
+ 'localBasePath' = __DIR__,
'remoteExtPath' = $remoteExtPath,
- 'localTemplateBasePath' = $localBasePath . '/templates',
+ 'localTemplateBasePath' = __DIR__ . '/templates',
'templates' = array( 'interstitial.hogan', 'zeroinfo.hogan' ),
// Use Mantle Resource Loader module
'class' = 'ResourceLoaderTemplateModule',
@@ -97,14 +96,14 @@
);
$wgResourceModules['zerobanner.special.scripts'] = array(
- 'localBasePath' = $localBasePath,
+ 'localBasePath' = __DIR__,
'remoteExtPath' = $remoteExtPath,
'targets' = array( 'mobile' ),
'scripts' = array( 'modules/forms.js' ),
);
$wgResourceModules['zerobanner.special.styles'] = array(
- 'localBasePath' = $localBasePath,
+ 'localBasePath' = __DIR__,
'remoteExtPath' = $remoteExtPath,
'targets' = array( 'mobile' ),
'styles' = array( 'modules/forms.css' ),
diff --git a/includes/PageRendering.php b/includes/PageRendering.php
index 3016b1d..88bcb95 100644
--- a/includes/PageRendering.php
+++ b/includes/PageRendering.php
@@ -70,12 +70,13 @@
}
/**
+* @param \ContextSource|\SpecialPage $ctxSrc
* @param string $id
* @param null|ZeroConfig $config
* @param bool $enabled
[MediaWiki-commits] [Gerrit] Enabled parameter now allows enabled range - change (mediawiki...ZeroBanner)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/163897
Change subject: Enabled parameter now allows enabled range
..
Enabled parameter now allows enabled range
Enabled can now be an array of two elements,
each being either a date or an integer.
If first is an integer, represents number of hours since now
If second is an integer, represents the duration (in hours)
Change-Id: I23839878a376e73324384b042e809d668f954308
---
M i18n/en.json
M i18n/qqq.json
M includes/ZeroConfig.php
3 files changed, 85 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroBanner
refs/changes/97/163897/1
diff --git a/i18n/en.json b/i18n/en.json
index de212ca..b4d27b6 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -43,6 +43,7 @@
zero-config-ipsets: Parameter \$1\ must be the name of an existing
ip set declared in \ipsets\ field,
zero-config-admins: Parameter \$1\ must be an existing user
account,
zero-config-proxies: Parameter \$1\ must be a list of proxies
supporting zero-rating for this carrier,
+ zero-config-enabled: Parameter \$1\ must be either true, false, or
a list with two timestamps. Timestamp could be in any time format. First array
can also be a number of hours after (before) now, and second value can be the
duration in hours.,
group-zeroadmin: Zero administrators,
group-zeroadmin-member: {{GENDER:$1|Zero administrator}},
grouppage-zeroadmin: {{ns:project}}:Zero administrators,
diff --git a/i18n/qqq.json b/i18n/qqq.json
index f8e90d0..0d1cc03 100644
--- a/i18n/qqq.json
+++ b/i18n/qqq.json
@@ -59,6 +59,7 @@
zero-config-ipsets: ipsets parameter validation error.
Parameters:\n* $1 - field name\n{{Related|Zeroconfig}},
zero-config-admins: Admins parameter validation error.
Parameters:\n* $1 - field name\n{{Related|Zeroconfig}},
zero-config-proxies: Proxies parameter validation error.
Parameters:\n* $1 - field name\n{{Related|Zeroconfig}},
+ zero-config-enabled: Enabled parameter validation error.
Parameters:\n* $1 - field name\n{{Related|Zeroconfig}},
group-zeroadmin: {{doc-group|zeroadmin}},
group-zeroadmin-member: {{doc-group|zeroadmin|member}},
grouppage-zeroadmin: {{doc-group|zeroadmin|page}},
diff --git a/includes/ZeroConfig.php b/includes/ZeroConfig.php
index 5801e43..caada8c 100644
--- a/includes/ZeroConfig.php
+++ b/includes/ZeroConfig.php
@@ -1,11 +1,16 @@
?php
namespace ZeroBanner;
+use DateInterval;
+use DateTime;
+use DateTimeZone;
use JsonConfig\JCObjContent;
use JsonConfig\JCUtils;
use JsonConfig\JCValidators;
use JsonConfig\JCValue;
+use MWTimestamp;
use stdClass;
+use TimestampException;
/**
* JSON Zero Config
@@ -25,6 +30,7 @@
private $mode = null;
private $ipset = null;
private $proxy = null;
+ private $enabledValue = false;
const ignoreNetwork = 0x1;
const ignoreDisabled = 0x2;
@@ -100,7 +106,7 @@
}
public function enabled() {
- return $this-config-enabled
$this-getDataWithDefaults()-enabled;
+ return $this-enabledValue
$this-getDataWithDefaults()-enabled;
}
public function name() {
@@ -205,7 +211,7 @@
// Optional comment
$this-testOptional( 'comment', '', $isString );
// Config is enabled
- $this-testOptional( 'enabled', true, $isBool );
+ $this-testOptional( 'enabled', true,
$this-getEnabledValidator() );
// Map of localized partner names
$this-test( 'name', self::getLangToStrValidator() );
// Map of localized banner texts with {{PARTNER}} placeholder
@@ -322,6 +328,76 @@
$this-testOptional( 'testInfoScreen', false, $isBool );
}
+ private function getEnabledValidator() {
+ $self = $this;
+ return function ( JCValue $jcv, array $path ) use ( $self ) {
+ if ( $jcv-isMissing() ) {
+ $jcv-setValue( true );
+ $self-setEnabled( true );
+ return true;
+ }
+ $v = $jcv-getValue();
+ if ( is_bool( $v ) ) {
+ $self-setEnabled( $v );
+ return true;
+ }
+ if ( JCUtils::isList( $v ) count( $v ) === 2 ) {
+ /** @var DateTime[] $range */
+ $range = array();
+ $now = null;
+ foreach ( $v as $time ) {
+ if ( is_int( $time ) ) {
+ // first digit - relative to
now, second
[MediaWiki-commits] [Gerrit] Added missing JsonFormat::parse() RELEASE NOTES, fixed docs - change (mediawiki/core)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/163335
Change subject: Added missing JsonFormat::parse() RELEASE NOTES, fixed docs
..
Added missing JsonFormat::parse() RELEASE NOTES, fixed docs
Change-Id: I9bfd67a5ca4ea1d399821549c7e63ffdecd56ad1
---
M RELEASE-NOTES-1.24
M includes/json/FormatJson.php
2 files changed, 9 insertions(+), 6 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/35/163335/1
diff --git a/RELEASE-NOTES-1.24 b/RELEASE-NOTES-1.24
index becc5a9..1f4fcfe 100644
--- a/RELEASE-NOTES-1.24
+++ b/RELEASE-NOTES-1.24
@@ -193,6 +193,7 @@
* (bug 69418) A MultiConfig implementation was added that supports fallback
to multiple Config instances.
* Update CSSJanus to v1.1.0.
+* Added FormatJson::parse() returning status with result or localized error
message
=== Bug fixes in 1.24 ===
* (bug 50572) MediaWiki:Blockip should support gender
diff --git a/includes/json/FormatJson.php b/includes/json/FormatJson.php
index 5565644..3430d25 100644
--- a/includes/json/FormatJson.php
+++ b/includes/json/FormatJson.php
@@ -123,14 +123,16 @@
}
/**
-* Decodes a JSON string.
+* Decodes a JSON string. It is recommended to use FormatJson::parse(),
which returns more comprehensive
+* result in case of an error.
*
* @param string $value The JSON string being decoded
* @param bool $assoc When true, returned objects will be converted
into associative arrays.
*
* @return mixed The value encoded in JSON in appropriate PHP type.
-* `null` is returned if the JSON cannot be decoded or if the encoded
data is deeper than
-* the recursion limit.
+* `null` is returned if $value represented `null`, if $value could not
be decoded,
+* or if the encoded data was deeper than the recursion limit.
+* Use FormatJson::parse() to distinguish between types of `null` and
to get proper error code.
*/
public static function decode( $value, $assoc = false ) {
return json_decode( $value, $assoc );
@@ -138,11 +140,11 @@
/**
* Decodes a JSON string.
+* Unlike FormatJson::decode(), if $value represents null value, it
will be properly decoded as valid.
*
* @param string $value The JSON string being decoded
-* @param int $options A bit field that allows FORCE_ASSOC, TRY_FIXING,
WRAP_RESULT
-* For backward compatibility, FORCE_ASSOC is set to 1 to match the
legacy 'true'
-* @return Status If good, the value is available in $result-getValue()
+* @param int $options A bit field that allows FORCE_ASSOC
+* @return Status If valid JSON, the value is available in
$result-getValue()
*/
public static function parse( $value, $options = 0 ) {
$assoc = ( $options self::FORCE_ASSOC ) !== 0;
--
To view, visit https://gerrit.wikimedia.org/r/163335
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9bfd67a5ca4ea1d399821549c7e63ffdecd56ad1
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] FormatJson::parse( TRY_FIXING ) - remove trailing commas - change (mediawiki/core)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/163344
Change subject: FormatJson::parse( TRY_FIXING ) - remove trailing commas
..
FormatJson::parse( TRY_FIXING ) - remove trailing commas
Removes trailing commas from json text when parsing
Solves very common cases like [1,2,3,]
Resulting status will be set to OK but not Good to warn caller
Change-Id: Ic0eb0a711da3ae578d6bb58d7474279d6845a4a7
---
M includes/json/FormatJson.php
M languages/i18n/en.json
M languages/i18n/qqq.json
M tests/phpunit/includes/json/FormatJsonTest.php
4 files changed, 68 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/44/163344/1
diff --git a/includes/json/FormatJson.php b/includes/json/FormatJson.php
index 5565644..2dbbc30 100644
--- a/includes/json/FormatJson.php
+++ b/includes/json/FormatJson.php
@@ -61,7 +61,14 @@
*
* @since 1.24
*/
- const FORCE_ASSOC = 1;
+ const FORCE_ASSOC = 0x1;
+
+ /**
+* If set, attempts to fix invalid json.
+*
+* @since 1.24
+*/
+ const TRY_FIXING = 0x2;
/**
* Regex that matches whitespace inside empty arrays and objects.
@@ -149,6 +156,28 @@
$result = json_decode( $value, $assoc );
$code = json_last_error();
+ if ( $code === JSON_ERROR_SYNTAX ( $options
self::TRY_FIXING ) !== 0 ) {
+ // The most common error is the trailing comma in a
list or an object.
+ // We cannot simply replace /,\s*[}\]]/ because it
could be inside a string value.
+ // But we could use the fact that JSON does not allow
multi-line string values,
+ // And remove trailing commas if they are et the end of
a line.
+ // JSON only allows 4 control characters: [ \t\r\n].
So we must not use '\s' for matching.
+ // Regex match ,]any non-quote chars\n or ,\n]
with optional spaces/tabs.
+ $count = 0;
+ $value =
+ preg_replace( '/,([
\t]*[}\]][^\r\n]*([\r\n]|$)|[ \t]*[\r\n][ \t\r\n]*[}\]])/', '$1',
+ $value, - 1, $count );
+ if ( $count 0 ) {
+ $result = json_decode( $value, $assoc );
+ if ( JSON_ERROR_NONE === json_last_error() ) {
+ // Report warning
+ $st = Status::newGood( $result );
+ $st-warning( wfMessage(
'json-warn-trailing-comma' )-numParams( $count ) );
+ return $st;
+ }
+ }
+ }
+
switch ( $code ) {
case JSON_ERROR_NONE:
return Status::newGood( $result );
diff --git a/languages/i18n/en.json b/languages/i18n/en.json
index e8a5c3c..7a1d2f5 100644
--- a/languages/i18n/en.json
+++ b/languages/i18n/en.json
@@ -3559,6 +3559,7 @@
mediastatistics-header-text: Textual,
mediastatistics-header-executable: Executables,
mediastatistics-header-archive: Compressed formats,
+ json-warn-trailing-comma: $1 trailing {{PLURAL:$1|comma was|commas
were}} removed from JSON,
json-error-unknown: There was a problem with the JSON. Error: $1,
json-error-depth: The maximum stack depth has been exceeded,
json-error-state-mismatch: Invalid or malformed JSON,
diff --git a/languages/i18n/qqq.json b/languages/i18n/qqq.json
index 2049ae5..96e7fac 100644
--- a/languages/i18n/qqq.json
+++ b/languages/i18n/qqq.json
@@ -3721,6 +3721,7 @@
mediastatistics-header-text: Header on [[Special:MediaStatistics]]
for file types that are in the text category. This includes simple text
formats, including plain text formats, json, csv, and xml. Source code of
compiled programming languages may be included here in the future, but isn't
currently.,
mediastatistics-header-executable: Header on
[[Special:MediaStatistics]] for file types that are in the executable category.
This includes things like source files for interpreted programming language
(Shell scripts, javascript, etc).,
mediastatistics-header-archive: Header on
[[Special:MediaStatistics]] for file types that are in the archive category.
Includes things like tar, zip, gzip etc.,
+ json-warn-trailing-comma: A warning message notifying that JSON text
was automatically corrected by removing erroneous commas.\n\nParameters:\n* $1
- number of commas that were removed,
json-error-unknown: User error message when there’s an unknown
error.\n{{Identical|Unknown error}}. This error is shown if we
[MediaWiki-commits] [Gerrit] FormatJson::parse( TRY_FIXING ) - remove trailing commas - change (mediawiki/core)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/163456
Change subject: FormatJson::parse( TRY_FIXING ) - remove trailing commas
..
FormatJson::parse( TRY_FIXING ) - remove trailing commas
Removes trailing commas from json text when parsing
Solves very common cases like [1,2,3,]
Resulting status will be set to OK but not Good to warn caller
Change-Id: Ic0eb0a711da3ae578d6bb58d7474279d6845a4a7
(cherry picked from commit 289d3e4f503fac5326e986f27e0b1d924c7f)
---
M includes/json/FormatJson.php
M languages/i18n/en.json
M languages/i18n/qqq.json
M tests/phpunit/includes/json/FormatJsonTest.php
4 files changed, 68 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/56/163456/1
diff --git a/includes/json/FormatJson.php b/includes/json/FormatJson.php
index 5565644..2dbbc30 100644
--- a/includes/json/FormatJson.php
+++ b/includes/json/FormatJson.php
@@ -61,7 +61,14 @@
*
* @since 1.24
*/
- const FORCE_ASSOC = 1;
+ const FORCE_ASSOC = 0x1;
+
+ /**
+* If set, attempts to fix invalid json.
+*
+* @since 1.24
+*/
+ const TRY_FIXING = 0x2;
/**
* Regex that matches whitespace inside empty arrays and objects.
@@ -149,6 +156,28 @@
$result = json_decode( $value, $assoc );
$code = json_last_error();
+ if ( $code === JSON_ERROR_SYNTAX ( $options
self::TRY_FIXING ) !== 0 ) {
+ // The most common error is the trailing comma in a
list or an object.
+ // We cannot simply replace /,\s*[}\]]/ because it
could be inside a string value.
+ // But we could use the fact that JSON does not allow
multi-line string values,
+ // And remove trailing commas if they are et the end of
a line.
+ // JSON only allows 4 control characters: [ \t\r\n].
So we must not use '\s' for matching.
+ // Regex match ,]any non-quote chars\n or ,\n]
with optional spaces/tabs.
+ $count = 0;
+ $value =
+ preg_replace( '/,([
\t]*[}\]][^\r\n]*([\r\n]|$)|[ \t]*[\r\n][ \t\r\n]*[}\]])/', '$1',
+ $value, - 1, $count );
+ if ( $count 0 ) {
+ $result = json_decode( $value, $assoc );
+ if ( JSON_ERROR_NONE === json_last_error() ) {
+ // Report warning
+ $st = Status::newGood( $result );
+ $st-warning( wfMessage(
'json-warn-trailing-comma' )-numParams( $count ) );
+ return $st;
+ }
+ }
+ }
+
switch ( $code ) {
case JSON_ERROR_NONE:
return Status::newGood( $result );
diff --git a/languages/i18n/en.json b/languages/i18n/en.json
index 01a826c..58236d3 100644
--- a/languages/i18n/en.json
+++ b/languages/i18n/en.json
@@ -3562,6 +3562,7 @@
mediastatistics-header-text: Textual,
mediastatistics-header-executable: Executables,
mediastatistics-header-archive: Compressed formats,
+ json-warn-trailing-comma: $1 trailing {{PLURAL:$1|comma was|commas
were}} removed from JSON,
json-error-unknown: There was a problem with the JSON. Error: $1,
json-error-depth: The maximum stack depth has been exceeded,
json-error-state-mismatch: Invalid or malformed JSON,
diff --git a/languages/i18n/qqq.json b/languages/i18n/qqq.json
index cfde0cd..31504bb 100644
--- a/languages/i18n/qqq.json
+++ b/languages/i18n/qqq.json
@@ -3722,6 +3722,7 @@
mediastatistics-header-text: Header on [[Special:MediaStatistics]]
for file types that are in the text category. This includes simple text
formats, including plain text formats, json, csv, and xml. Source code of
compiled programming languages may be included here in the future, but isn't
currently.,
mediastatistics-header-executable: Header on
[[Special:MediaStatistics]] for file types that are in the executable category.
This includes things like source files for interpreted programming language
(Shell scripts, javascript, etc).,
mediastatistics-header-archive: Header on
[[Special:MediaStatistics]] for file types that are in the archive category.
Includes things like tar, zip, gzip etc.,
+ json-warn-trailing-comma: A warning message notifying that JSON text
was automatically corrected by removing erroneous commas.\n\nParameters:\n* $1
- number of commas that were removed,
json-error-unknown: User error message when there’s an unknown
[MediaWiki-commits] [Gerrit] top 5 languages - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/163128
Change subject: top 5 languages
..
top 5 languages
Change-Id: I9325f16c703d5908b6490cd5b5963e8e0cf030ec
---
M scripts/weblogs.py
1 file changed, 40 insertions(+), 20 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/28/163128/1
diff --git a/scripts/weblogs.py b/scripts/weblogs.py
index 3806ecd..8a0e0cd 100644
--- a/scripts/weblogs.py
+++ b/scripts/weblogs.py
@@ -5,9 +5,8 @@
import collections
import sys
from pandas import read_table, pivot_table
-from pandas.core.frame import DataFrame
+from pandas.core.frame import DataFrame, Series
import numpy as np
-import api
from logprocessor import *
@@ -284,33 +283,54 @@
allData = DataFrame(stats, columns=columnHeaders11)
# filter type==DATA
-df = allData[allData['type'] == 'DATA']
-# filter out last date
-lastDate = df.date.max()
-df = df[df.date lastDate]
-# create an artificial yes/opera value
-opera = df[(df.via == 'OPERA') (df.iszero == 'yes')]
-opera['iszero'] = 'opera'
-df = df.append(opera)
+allData = allData[allData['type'] == 'DATA']
+# filter out last date
+lastDate = allData.date.max()
+df = allData[allData.date lastDate]
xcs = list(df.xcs.unique())
+wiki = self.getWiki()
for id in xcs:
+xcsDf = df[df.xcs == id]
+
+# create an artificial yes/opera value
+opera = xcsDf[(xcsDf.via == 'OPERA') (xcsDf.iszero == 'yes')]
+opera['str'] = 'zero-opera'
+
+yes = xcsDf[xcsDf.iszero == 'yes']
+yes['str'] = 'zero-all'
+
+no = xcsDf[xcsDf.iszero == 'no']
+no['str'] = 'non-zero'
+
+combined = opera.append(yes).append(no)
+
s = StringIO.StringIO()
-pivot_table(df[df.xcs == id], 'count', ['date', 'iszero'],
aggfunc=np.sum).to_csv(s, header=True)
-result = s.getvalue()
+pivot_table(combined, 'count', ['date', 'str'],
aggfunc=np.sum).to_csv(s, header=False)
+result = 'date,iszero,count\n' + s.getvalue()
-# sortColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'iszero']
-# outColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'iszero', 'count']
-# xcsData = df[df.xcs == id].sort(columns=sortColumns)
-# result =
xcsData.sort(columns=sortColumns).to_csv(columns=outColumns, index=False)
-
-wiki = self.getWiki()
wiki(
'edit',
title='RawData:' + id,
+summary='refreshing data',
+text=result,
+token=wiki.token()
+)
+
+
+byLang = pivot_table(xcsDf, 'count', ['lang'],
aggfunc=np.sum).order('count', ascending=False)
+top = byLang.head(5)
+other = byLang.sum() - top.sum()
+s = StringIO.StringIO()
+Series.to_csv(top, s)
+result = 'lang,count\n' + s.getvalue() + ('other,%d\n' % other)
+
+wiki(
+'edit',
+title='RawData:' + id + '-langTotal',
summary='refreshing data',
text=result,
token=wiki.token()
@@ -352,5 +372,5 @@
self.generateGraphData()
if __name__ == '__main__':
-# WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
-WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
+WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
+# WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
--
To view, visit https://gerrit.wikimedia.org/r/163128
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9325f16c703d5908b6490cd5b5963e8e0cf030ec
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] top 5 languages - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: top 5 languages
..
top 5 languages
Change-Id: I9325f16c703d5908b6490cd5b5963e8e0cf030ec
---
M scripts/weblogs.py
1 file changed, 40 insertions(+), 20 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/weblogs.py b/scripts/weblogs.py
index 3806ecd..8a0e0cd 100644
--- a/scripts/weblogs.py
+++ b/scripts/weblogs.py
@@ -5,9 +5,8 @@
import collections
import sys
from pandas import read_table, pivot_table
-from pandas.core.frame import DataFrame
+from pandas.core.frame import DataFrame, Series
import numpy as np
-import api
from logprocessor import *
@@ -284,33 +283,54 @@
allData = DataFrame(stats, columns=columnHeaders11)
# filter type==DATA
-df = allData[allData['type'] == 'DATA']
-# filter out last date
-lastDate = df.date.max()
-df = df[df.date lastDate]
-# create an artificial yes/opera value
-opera = df[(df.via == 'OPERA') (df.iszero == 'yes')]
-opera['iszero'] = 'opera'
-df = df.append(opera)
+allData = allData[allData['type'] == 'DATA']
+# filter out last date
+lastDate = allData.date.max()
+df = allData[allData.date lastDate]
xcs = list(df.xcs.unique())
+wiki = self.getWiki()
for id in xcs:
+xcsDf = df[df.xcs == id]
+
+# create an artificial yes/opera value
+opera = xcsDf[(xcsDf.via == 'OPERA') (xcsDf.iszero == 'yes')]
+opera['str'] = 'zero-opera'
+
+yes = xcsDf[xcsDf.iszero == 'yes']
+yes['str'] = 'zero-all'
+
+no = xcsDf[xcsDf.iszero == 'no']
+no['str'] = 'non-zero'
+
+combined = opera.append(yes).append(no)
+
s = StringIO.StringIO()
-pivot_table(df[df.xcs == id], 'count', ['date', 'iszero'],
aggfunc=np.sum).to_csv(s, header=True)
-result = s.getvalue()
+pivot_table(combined, 'count', ['date', 'str'],
aggfunc=np.sum).to_csv(s, header=False)
+result = 'date,iszero,count\n' + s.getvalue()
-# sortColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'iszero']
-# outColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'iszero', 'count']
-# xcsData = df[df.xcs == id].sort(columns=sortColumns)
-# result =
xcsData.sort(columns=sortColumns).to_csv(columns=outColumns, index=False)
-
-wiki = self.getWiki()
wiki(
'edit',
title='RawData:' + id,
+summary='refreshing data',
+text=result,
+token=wiki.token()
+)
+
+
+byLang = pivot_table(xcsDf, 'count', ['lang'],
aggfunc=np.sum).order('count', ascending=False)
+top = byLang.head(5)
+other = byLang.sum() - top.sum()
+s = StringIO.StringIO()
+Series.to_csv(top, s)
+result = 'lang,count\n' + s.getvalue() + ('other,%d\n' % other)
+
+wiki(
+'edit',
+title='RawData:' + id + '-langTotal',
summary='refreshing data',
text=result,
token=wiki.token()
@@ -352,5 +372,5 @@
self.generateGraphData()
if __name__ == '__main__':
-# WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
-WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
+WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
+# WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
--
To view, visit https://gerrit.wikimedia.org/r/163128
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9325f16c703d5908b6490cd5b5963e8e0cf030ec
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
Gerrit-Reviewer: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] AllEnabled data aggregation, filter by wikipedia only - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/163220
Change subject: AllEnabled data aggregation, filter by wikipedia only
..
AllEnabled data aggregation, filter by wikipedia only
Change-Id: I1298032af723f2f9e1aa78dc955f604695a09e3d
---
M scripts/weblogs.py
1 file changed, 46 insertions(+), 26 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/20/163220/1
diff --git a/scripts/weblogs.py b/scripts/weblogs.py
index 8a0e0cd..5f98aa7 100644
--- a/scripts/weblogs.py
+++ b/scripts/weblogs.py
@@ -20,8 +20,8 @@
stats[key] = 1
-columnHeaders10 =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
-columnHeaders11 =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,iszero,count'.split(',')
+columnHdrCache =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
+columnHdrResult =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,iszero,ison,count'.split(',')
validSubDomains = {'m', 'zero', 'mobile', 'wap'}
validHttpCode = {'200', '304'}
@@ -210,7 +210,7 @@
# Valid request!
addStat(stats, dt, 'DATA', xcs, via, ipset, https, lang,
subdomain, site)
-writeData(statFile, [list(k) + [v] for k, v in stats.iteritems()],
columnHeaders10)
+writeData(statFile, [list(k) + [v] for k, v in stats.iteritems()],
columnHdrCache)
def combineStats(self):
safePrint('Combine stat files')
@@ -221,7 +221,7 @@
for f in os.listdir(self.pathCache):
if not self.statFileRe.match(f):
continue
-for vals in readData(os.path.join(self.pathCache, f),
columnHeaders10):
+for vals in readData(os.path.join(self.pathCache, f),
columnHdrCache):
# 0 12 3 4 56 78
9
# 2014-07-25 DATA 250-99 DIRECT default http ru zero
wikipedia 1000
if len(vals) != 10:
@@ -240,30 +240,38 @@
if xcs == '404-01b':
vals[2] = xcs = '404-01'
vals[4] = ipset = 'b'
+
+isZero = ''
+isOn = ''
if typ == 'DATA':
dt = datetime.strptime(dt, '%Y-%m-%d')
site2 = subdomain + '.' + site
isZero = False
+isEnabled = False
for conf in configs[xcs]:
langs = conf.languages
sites = conf.sites
-if conf['from'] = dt conf.before and \
-(conf.https or https == u'http') and \
-(True == langs or lang in langs) and \
-(True == sites or site2 in sites) and \
-(dt ignoreViaBefore or via in conf.via)
and \
-(ipset in conf.ipsets):
-isZero = True
-break
-vals[9] = u'yes' if isZero else u'no'
-else:
-vals[9] = ''
+if conf['from'] = dt conf.before:
+if 'enabled' not in conf or conf.enabled:
+isEnabled = True
+if (conf.https or https == u'http') and \
+(True == langs or lang in langs) and \
+(True == sites or site2 in sites) and \
+(dt ignoreViaBefore or via in
conf.via) and \
+(ipset in conf.ipsets):
+isZero = True
+break
+isZero = u'yes' if isZero else u'no'
+isOn = u'on' if isEnabled else u'off'
+
+vals[9] = isZero
+vals[10] = isOn
else:
# X-CS does not exist, ignore it
error = 'xcs'
if error:
-vals = (vals[0], 'ERR', '000-00', 'ERR', 'ERR', 'http',
'', error, '', '')
+vals = (vals[0], 'ERR', '000-00', 'ERR', 'ERR', 'http',
'', error, '', '', '')
key = tuple(vals)
stats[key] += int(count)
@@ -271,28 +279,40 @@
# convert {a|b|c:count,...} into [[a,b,c,count],...]
stats = [list(k) + [v] for k, v in stats.iteritems()]
-writeData(self.combinedFile, stats, columnHeaders11)
+writeData(self.combinedFile, stats, columnHdrResult)
return stats
def generateGraphData(self, stats=None):
[MediaWiki-commits] [Gerrit] AllEnabled data aggregation, filter by wikipedia only - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: AllEnabled data aggregation, filter by wikipedia only
..
AllEnabled data aggregation, filter by wikipedia only
Change-Id: I1298032af723f2f9e1aa78dc955f604695a09e3d
---
M scripts/weblogs.py
1 file changed, 46 insertions(+), 26 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/weblogs.py b/scripts/weblogs.py
index 8a0e0cd..5f98aa7 100644
--- a/scripts/weblogs.py
+++ b/scripts/weblogs.py
@@ -20,8 +20,8 @@
stats[key] = 1
-columnHeaders10 =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
-columnHeaders11 =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,iszero,count'.split(',')
+columnHdrCache =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
+columnHdrResult =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,iszero,ison,count'.split(',')
validSubDomains = {'m', 'zero', 'mobile', 'wap'}
validHttpCode = {'200', '304'}
@@ -210,7 +210,7 @@
# Valid request!
addStat(stats, dt, 'DATA', xcs, via, ipset, https, lang,
subdomain, site)
-writeData(statFile, [list(k) + [v] for k, v in stats.iteritems()],
columnHeaders10)
+writeData(statFile, [list(k) + [v] for k, v in stats.iteritems()],
columnHdrCache)
def combineStats(self):
safePrint('Combine stat files')
@@ -221,7 +221,7 @@
for f in os.listdir(self.pathCache):
if not self.statFileRe.match(f):
continue
-for vals in readData(os.path.join(self.pathCache, f),
columnHeaders10):
+for vals in readData(os.path.join(self.pathCache, f),
columnHdrCache):
# 0 12 3 4 56 78
9
# 2014-07-25 DATA 250-99 DIRECT default http ru zero
wikipedia 1000
if len(vals) != 10:
@@ -240,30 +240,38 @@
if xcs == '404-01b':
vals[2] = xcs = '404-01'
vals[4] = ipset = 'b'
+
+isZero = ''
+isOn = ''
if typ == 'DATA':
dt = datetime.strptime(dt, '%Y-%m-%d')
site2 = subdomain + '.' + site
isZero = False
+isEnabled = False
for conf in configs[xcs]:
langs = conf.languages
sites = conf.sites
-if conf['from'] = dt conf.before and \
-(conf.https or https == u'http') and \
-(True == langs or lang in langs) and \
-(True == sites or site2 in sites) and \
-(dt ignoreViaBefore or via in conf.via)
and \
-(ipset in conf.ipsets):
-isZero = True
-break
-vals[9] = u'yes' if isZero else u'no'
-else:
-vals[9] = ''
+if conf['from'] = dt conf.before:
+if 'enabled' not in conf or conf.enabled:
+isEnabled = True
+if (conf.https or https == u'http') and \
+(True == langs or lang in langs) and \
+(True == sites or site2 in sites) and \
+(dt ignoreViaBefore or via in
conf.via) and \
+(ipset in conf.ipsets):
+isZero = True
+break
+isZero = u'yes' if isZero else u'no'
+isOn = u'on' if isEnabled else u'off'
+
+vals[9] = isZero
+vals[10] = isOn
else:
# X-CS does not exist, ignore it
error = 'xcs'
if error:
-vals = (vals[0], 'ERR', '000-00', 'ERR', 'ERR', 'http',
'', error, '', '')
+vals = (vals[0], 'ERR', '000-00', 'ERR', 'ERR', 'http',
'', error, '', '', '')
key = tuple(vals)
stats[key] += int(count)
@@ -271,28 +279,40 @@
# convert {a|b|c:count,...} into [[a,b,c,count],...]
stats = [list(k) + [v] for k, v in stats.iteritems()]
-writeData(self.combinedFile, stats, columnHeaders11)
+writeData(self.combinedFile, stats, columnHdrResult)
return stats
def generateGraphData(self, stats=None):
safePrint('Generating data files to %s' % self.pathGraphs)
+wiki =
[MediaWiki-commits] [Gerrit] Recursivel git-update - change (mediawiki/vagrant)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/163293
Change subject: Recursivel git-update
..
Recursivel git-update
Recursivelly pulls extensions/ and skins/ directories
Change-Id: I55708ce6f09e3fcfa54375f8b33b6638ba3257b6
---
M puppet/modules/mediawiki/files/run-git-update
1 file changed, 20 insertions(+), 14 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/93/163293/1
diff --git a/puppet/modules/mediawiki/files/run-git-update
b/puppet/modules/mediawiki/files/run-git-update
index be3faed..80b38a6 100644
--- a/puppet/modules/mediawiki/files/run-git-update
+++ b/puppet/modules/mediawiki/files/run-git-update
@@ -9,6 +9,19 @@
UPDATE_ERRORS=( )
+function recursivePull {
+ for f in *; do
+if [[ -d $f ]]; then
+ pushd $f /dev/null
+ if [[ -d .git ]]; then
+pull
+ fi
+ recursivePull
+ popd /dev/null
+fi
+ done
+}
+
function pull {
echo -e \e[36m== Updating $(pwd) ...\e[0m
branch=$(expr $(git symbolic-ref HEAD) : 'refs/heads/\(.*\)')
@@ -52,21 +65,14 @@
cd $MW_INSTALL_PATH
pull
-for f in extensions/*; do
- if [[ -d $f ]] [[ -d $f/.git ]]; then
-pushd $f /dev/null
-pull
-popd /dev/null
- fi
-done
+pushd extensions
+recursivePull
+popd /dev/null
-for f in skins/*; do
- if [[ -d $f ]] [[ -d $f/.git ]]; then
-pushd $f /dev/null
-pull
-popd /dev/null
- fi
-done
+pushd skins
+recursivePull
+popd /dev/null
+
echo -e \e[36m== Updating composer dependencies ...\e[0m
composer update --no-interaction --optimize-autoloader
--
To view, visit https://gerrit.wikimedia.org/r/163293
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I55708ce6f09e3fcfa54375f8b33b6638ba3257b6
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Added private-wiki-only lua extensions - change (mediawiki...ZeroPortal)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/162850
Change subject: Added private-wiki-only lua extensions
..
Added private-wiki-only lua extensions
* Several functions that expose MW API and access to request object
* Abilitiy to override output to raw format
* global var that controls all this $wgEnableSpecialLuaAccess
Change-Id: Ifecbbb6e989f1ca510fedeb42cd90d18ea218ff9
---
M ZeroPortal.php
A includes/LuaLibrary.lua
A includes/LuaLibrary.php
M includes/PortalSpecialPage.php
4 files changed, 354 insertions(+), 8 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroPortal
refs/changes/50/162850/1
diff --git a/ZeroPortal.php b/ZeroPortal.php
index b9249ad..a75eb5e 100644
--- a/ZeroPortal.php
+++ b/ZeroPortal.php
@@ -31,20 +31,22 @@
'url' = 'https://www.mediawiki.org/wiki/Extension:ZeroPortal',
);
-$cwd = __DIR__ . DIRECTORY_SEPARATOR;
-$wgMessagesDirs['ZeroPortal'] = $cwd . 'i18n';
-$wgExtensionMessagesFiles['ZeroPortal'] = $cwd . 'ZeroPortal.i18n.php';
+$zpDir = __DIR__ . DIRECTORY_SEPARATOR;
+$wgMessagesDirs['ZeroPortal'] = $zpDir . 'i18n';
+$wgExtensionMessagesFiles['ZeroPortal'] = $zpDir . 'ZeroPortal.i18n.php';
// autoload extension classes
-$cwd .= 'includes' . DIRECTORY_SEPARATOR;
+$zpDir .= 'includes' . DIRECTORY_SEPARATOR;
foreach ( array(
'ApiZeroPortal',
'ConfigPageHooks',
- 'ZeroConfigView',
+ 'LuaLibrary',
'PortalSpecialPage',
+ 'ZeroConfigView',
) as $key = $class ) {
- $wgAutoloadClasses['ZeroPortal\\' . ( is_string( $key ) ? $key : $class
)] = $cwd . $class . '.php';
+ $wgAutoloadClasses['ZeroPortal\\' . ( is_string( $key ) ? $key : $class
)] = $zpDir . $class . '.php';
}
+unset( $zpDir );
$wgResourceModules['zeroportal.config'] = array(
'dependencies' = 'zerobanner.config.styles', // defined in ZeroBanner
ext, used for sample banners
@@ -56,6 +58,12 @@
$wgAPIModules['zeroportal'] = 'ZeroPortal\ApiZeroPortal';
$wgHooks['BeforePageDisplay'][] =
'ZeroPortal\ConfigPageHooks::onBeforePageDisplay';
+$wgHooks['ScribuntoExternalLibraries'][] = function( $engine, array
$extraLibraries ) {
+ if( $engine == 'lua' ) {
+ $extraLibraries['mw.zeroportal'] = 'ZeroPortal\LuaLibrary';
+ }
+ return true;
+};
// Set our own view class for the JsonZeroConfig model, and configure local
storage
$wgJsonConfigModels['JsonZeroConfig']['view'] = 'ZeroPortal\ZeroConfigView';
@@ -66,3 +74,4 @@
$wgExtensionMessagesFiles['ZeroPortalAlias'] = __DIR__ . DIRECTORY_SEPARATOR .
'ZeroPortal.alias.php';
$wgZeroPortalImpersonateUser = false;
+$wgEnableSpecialLuaAccess = false;
diff --git a/includes/LuaLibrary.lua b/includes/LuaLibrary.lua
new file mode 100644
index 000..21a7d47
--- /dev/null
+++ b/includes/LuaLibrary.lua
@@ -0,0 +1,66 @@
+local p = {}
+local php
+
+function p.api( data )
+return php.api( data )
+end
+
+function p.getAccountsForUser()
+return php.getAccountsForUser()
+end
+
+function p.getCheck( name )
+return php.getCheck( name )
+end
+
+function p.getUsername()
+return php.getUsername()
+end
+
+function p.getVal( name, default )
+return php.getVal( name, default )
+end
+
+function p.isAdmin()
+return php.isAdmin()
+end
+
+function p.jsonDecode( value )
+return php.jsonDecode( value )
+end
+
+function p.jsonEncode( value, escapeHtml )
+return php.jsonEncode( value, escapeHtml )
+end
+
+function p.register()
+return php.register()
+end
+
+function p.setCache( mode, seconds )
+return php.setCache( mode, seconds )
+end
+
+function p.setResult( result, mime )
+return php.setResult( result, mime )
+end
+
+function p.wasPosted()
+return php.wasPosted()
+end
+
+
+function p.setupInterface( options )
+ -- Boilerplate
+ p.setupInterface = nil
+ php = mw_interface
+ mw_interface = nil
+
+ -- Register this library in the mw global
+ mw = mw or {}
+ mw.zeroportal = p
+
+ package.loaded['mw.zeroportal'] = p
+end
+
+return p
diff --git a/includes/LuaLibrary.php b/includes/LuaLibrary.php
new file mode 100644
index 000..e7b18ac
--- /dev/null
+++ b/includes/LuaLibrary.php
@@ -0,0 +1,257 @@
+?php
+
+namespace ZeroPortal;
+
+use ApiMain;
+use DerivativeContext;
+use FormatJson;
+use Scribunto_LuaError;
+use Scribunto_LuaLibraryBase;
+use IContextSource;
+use User;
+use ZeroBanner\ZeroConfig;
+
+class LuaLibrary extends Scribunto_LuaLibraryBase {
+
+ const luaNamespace = 'mw.zeroportal.';
+
+ /** @var IContextSource */
+ private static $context = null;
+
+ /** @var mixed */
+ private static $result = null;
+
+ /** @var string */
+ private static $mime = 'text/plain';
+
+ /** @var int */
+ private static $cacheMaxAge = 0;
+
+ /**
[MediaWiki-commits] [Gerrit] Enable ZeroPortal lua extensions - change (operations/mediawiki-config)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/162855
Change subject: Enable ZeroPortal lua extensions
..
Enable ZeroPortal lua extensions
(prevents meta from having the same)
Change-Id: Idaede28b76f92c3028ab0c3d975f9c16d18227c4
---
M wmf-config/CommonSettings.php
1 file changed, 1 insertion(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config
refs/changes/55/162855/1
diff --git a/wmf-config/CommonSettings.php b/wmf-config/CommonSettings.php
index a5fad0b..5988397 100644
--- a/wmf-config/CommonSettings.php
+++ b/wmf-config/CommonSettings.php
@@ -2737,6 +2737,7 @@
$wgGroupPermissions['zeroscriptips'] =
$wgGroupPermissions['zeroscriptips'] + $zpUserRights;
$wgZeroPortalImpersonateUser = 'Impersonator';
+ $wgZeroPortalEnableLuaExt = true;
unset( $zpUserRights );
} else {
--
To view, visit https://gerrit.wikimedia.org/r/162855
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Idaede28b76f92c3028ab0c3d975f9c16d18227c4
Gerrit-PatchSet: 1
Gerrit-Project: operations/mediawiki-config
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] opera value in graphs - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/162970
Change subject: opera value in graphs
..
opera value in graphs
Change-Id: Ib86e5bf5ce508287695298230c48b38457e214a1
---
M scripts/api.py
A scripts/countries.py
M scripts/logprocessor.py
A scripts/manual.py
M scripts/operaips.py
M scripts/weblogs.py
6 files changed, 158 insertions(+), 27 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/70/162970/1
diff --git a/scripts/api.py b/scripts/api.py
index e3e6312..4037aeb 100644
--- a/scripts/api.py
+++ b/scripts/api.py
@@ -5,6 +5,12 @@
import os
import sys
+PY3 = sys.version_info[0] == 3
+if PY3:
+string_types = str,
+else:
+string_types = basestring,
+
try:
import urllib.parse as urlparse
except ImportError:
@@ -61,12 +67,14 @@
super(ApiError, self).__init__('Pages modified during iteration', data)
-def parseJson(result):
-# Our servers still have requests 0.8.2 ... :(
-if hasattr(result.__class__, 'json'):
-return result.json(object_hook=AttrDict)
+def parseJson(value):
+if isinstance(value, string_types):
+return json.loads(value, object_hook=AttrDict)
+elif hasattr(value.__class__, 'json'):
+return value.json(object_hook=AttrDict)
else:
-return json.loads(result.content, object_hook=AttrDict)
+# Our servers still have requests 0.8.2 ... :(
+return json.loads(value.content, object_hook=AttrDict)
class Site(object):
diff --git a/scripts/countries.py b/scripts/countries.py
new file mode 100644
index 000..d4e0f67
--- /dev/null
+++ b/scripts/countries.py
@@ -0,0 +1,36 @@
+# coding=utf-8
+import collections
+import itertools
+from pandas import json
+
+from logprocessor import *
+
+class CountriesScript(ScriptProcessor):
+def __init__(self, settingsFile='settings/countries.json'):
+super(CountriesScript, self).__init__(settingsFile, 'countries')
+
+def run(self):
+enabled = collections.defaultdict(int)
+disabled = collections.defaultdict(int)
+wiki = self.getWiki()
+for res in wiki.queryPages(generator='allpages', gaplimit='max',
gapnamespace='480', prop='revisions', rvprop='content'):
+data = api.parseJson(res.revisions[0]['*'])
+if 'country' in data:
+# enabled by default
+if 'enabled' not in data or data.enabled:
+enabled[data.country] += 1
+else:
+disabled[data.country] += 1
+for state, vals in {'Enabled':enabled,
'Disabled':disabled}.iteritems():
+text = json.dumps([{code:k, val:v} for k,v in
vals.iteritems()])
+wiki(
+'edit',
+title='Data:Json:StatsByCountry-' + state,
+summary='updating - %d countries' % (sum(vals.values())),
+text=text,
+token=wiki.token()
+)
+
+
+if __name__ == '__main__':
+CountriesScript().safeRun()
diff --git a/scripts/logprocessor.py b/scripts/logprocessor.py
index 5718e57..36c7548 100644
--- a/scripts/logprocessor.py
+++ b/scripts/logprocessor.py
@@ -8,6 +8,7 @@
from unidecode import unidecode
from api import AttrDict
+import api
from utils import CsvUnicodeWriter, CsvUnicodeReader
@@ -85,6 +86,7 @@
class ScriptProcessor(object):
def __init__(self, settingsFile, pathSuffix):
+self._wiki = None
self.dateFormat = '%Y-%m-%d'
self.dateTimeFormat = '%Y-%m-%d %H:%M:%S'
@@ -179,6 +181,14 @@
def onSettingsLoaded(self):
pass
+def getWiki(self):
+if not self._wiki:
+self._wiki = api.wikimedia('zero', 'wikimedia', 'https')
+if self.proxy:
+self._wiki.session.proxies = {'http': 'http://%s:%d' %
(self.proxy, self.proxyPort)}
+self._wiki.login(self.settings.apiUsername,
self.settings.apiPassword)
+return self._wiki
+
# noinspection PyMethodMayBeStatic
def formatDate(self, value, dateFormat):
return value.strftime(dateFormat) if isinstance(value, datetime) else
value
diff --git a/scripts/manual.py b/scripts/manual.py
new file mode 100644
index 000..8113e44
--- /dev/null
+++ b/scripts/manual.py
@@ -0,0 +1,81 @@
+# coding=utf-8
+
+from logprocessor import *
+
+map = {
+414-06: MM,
+520-18: TH,
+631-02: AO,
+426-04: BH,
+470-01: BD,
+470-03: BD,
+470-07: BD,
+652-02: BW,
+456-02: KH,
+624-02: CM,
+623-03: CF,
+630-86: CG,
+620-01: GH,
+630-03: GH,
+404-01: IN,
+510-11: ID,
+612-03: CI,
+416-03: JO,
+401-01: KZ,
+639-02: KE,
+639-03: KE,
+639-07: KE,
+293-41: XK,
+437-01: KG,
+646-02: MG,
+502-13: MY,
+502-16: MY,
+428-98: MN,
+297-01: ME,
+604-00: MA,
+429-02: NP,
+614-04: NE,
+
[MediaWiki-commits] [Gerrit] opera value in graphs - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: opera value in graphs
..
opera value in graphs
Change-Id: Ib86e5bf5ce508287695298230c48b38457e214a1
---
M scripts/api.py
A scripts/countries.py
M scripts/logprocessor.py
A scripts/manual.py
M scripts/operaips.py
M scripts/weblogs.py
6 files changed, 158 insertions(+), 27 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/api.py b/scripts/api.py
index e3e6312..4037aeb 100644
--- a/scripts/api.py
+++ b/scripts/api.py
@@ -5,6 +5,12 @@
import os
import sys
+PY3 = sys.version_info[0] == 3
+if PY3:
+string_types = str,
+else:
+string_types = basestring,
+
try:
import urllib.parse as urlparse
except ImportError:
@@ -61,12 +67,14 @@
super(ApiError, self).__init__('Pages modified during iteration', data)
-def parseJson(result):
-# Our servers still have requests 0.8.2 ... :(
-if hasattr(result.__class__, 'json'):
-return result.json(object_hook=AttrDict)
+def parseJson(value):
+if isinstance(value, string_types):
+return json.loads(value, object_hook=AttrDict)
+elif hasattr(value.__class__, 'json'):
+return value.json(object_hook=AttrDict)
else:
-return json.loads(result.content, object_hook=AttrDict)
+# Our servers still have requests 0.8.2 ... :(
+return json.loads(value.content, object_hook=AttrDict)
class Site(object):
diff --git a/scripts/countries.py b/scripts/countries.py
new file mode 100644
index 000..d4e0f67
--- /dev/null
+++ b/scripts/countries.py
@@ -0,0 +1,36 @@
+# coding=utf-8
+import collections
+import itertools
+from pandas import json
+
+from logprocessor import *
+
+class CountriesScript(ScriptProcessor):
+def __init__(self, settingsFile='settings/countries.json'):
+super(CountriesScript, self).__init__(settingsFile, 'countries')
+
+def run(self):
+enabled = collections.defaultdict(int)
+disabled = collections.defaultdict(int)
+wiki = self.getWiki()
+for res in wiki.queryPages(generator='allpages', gaplimit='max',
gapnamespace='480', prop='revisions', rvprop='content'):
+data = api.parseJson(res.revisions[0]['*'])
+if 'country' in data:
+# enabled by default
+if 'enabled' not in data or data.enabled:
+enabled[data.country] += 1
+else:
+disabled[data.country] += 1
+for state, vals in {'Enabled':enabled,
'Disabled':disabled}.iteritems():
+text = json.dumps([{code:k, val:v} for k,v in
vals.iteritems()])
+wiki(
+'edit',
+title='Data:Json:StatsByCountry-' + state,
+summary='updating - %d countries' % (sum(vals.values())),
+text=text,
+token=wiki.token()
+)
+
+
+if __name__ == '__main__':
+CountriesScript().safeRun()
diff --git a/scripts/logprocessor.py b/scripts/logprocessor.py
index 5718e57..36c7548 100644
--- a/scripts/logprocessor.py
+++ b/scripts/logprocessor.py
@@ -8,6 +8,7 @@
from unidecode import unidecode
from api import AttrDict
+import api
from utils import CsvUnicodeWriter, CsvUnicodeReader
@@ -85,6 +86,7 @@
class ScriptProcessor(object):
def __init__(self, settingsFile, pathSuffix):
+self._wiki = None
self.dateFormat = '%Y-%m-%d'
self.dateTimeFormat = '%Y-%m-%d %H:%M:%S'
@@ -179,6 +181,14 @@
def onSettingsLoaded(self):
pass
+def getWiki(self):
+if not self._wiki:
+self._wiki = api.wikimedia('zero', 'wikimedia', 'https')
+if self.proxy:
+self._wiki.session.proxies = {'http': 'http://%s:%d' %
(self.proxy, self.proxyPort)}
+self._wiki.login(self.settings.apiUsername,
self.settings.apiPassword)
+return self._wiki
+
# noinspection PyMethodMayBeStatic
def formatDate(self, value, dateFormat):
return value.strftime(dateFormat) if isinstance(value, datetime) else
value
diff --git a/scripts/manual.py b/scripts/manual.py
new file mode 100644
index 000..8113e44
--- /dev/null
+++ b/scripts/manual.py
@@ -0,0 +1,81 @@
+# coding=utf-8
+
+from logprocessor import *
+
+map = {
+414-06: MM,
+520-18: TH,
+631-02: AO,
+426-04: BH,
+470-01: BD,
+470-03: BD,
+470-07: BD,
+652-02: BW,
+456-02: KH,
+624-02: CM,
+623-03: CF,
+630-86: CG,
+620-01: GH,
+630-03: GH,
+404-01: IN,
+510-11: ID,
+612-03: CI,
+416-03: JO,
+401-01: KZ,
+639-02: KE,
+639-03: KE,
+639-07: KE,
+293-41: XK,
+437-01: KG,
+646-02: MG,
+502-13: MY,
+502-16: MY,
+428-98: MN,
+297-01: ME,
+604-00: MA,
+429-02: NP,
+614-04: NE,
+621-20: NG,
+621-30: NG,
+410-01: PK,
+410-06: PK,
[MediaWiki-commits] [Gerrit] timer on finally - change (mediawiki...Graph)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/162663
Change subject: timer on finally
..
timer on finally
Change-Id: If038a2de87095d3c86ad9987e1acd6bb78686071
---
M js/graph.editor.js
1 file changed, 6 insertions(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Graph
refs/changes/63/162663/1
diff --git a/js/graph.editor.js b/js/graph.editor.js
index ce3fa7c..303eec6 100644
--- a/js/graph.editor.js
+++ b/js/graph.editor.js
@@ -14,12 +14,13 @@
var spec = $.parseJSON(content);
if (spec === null) return;
if (this.hasAttribute('spec'))
this.removeAttribute('spec');
- vg.parse.spec(spec, function(chart) {
chart({el:el}).update(); });
+ vg.parse.spec(spec, function (chart) {
+ chart({el: el}).update();
+ });
});
- } catch(err) {
- // Ignore all errors
+ } finally {
+ // FIXME: This should be done on data modification, not
on timer
+ setTimeout(arguments.callee, 300);
}
- // FIXME: This should be done on data modification, not on timer
- setTimeout(arguments.callee, 300);
})();
} ( jQuery ) );
--
To view, visit https://gerrit.wikimedia.org/r/162663
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If038a2de87095d3c86ad9987e1acd6bb78686071
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Graph
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated Graph, ZeroPortal, ZeroBanner, JsonConfig to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/162667 Change subject: Updated Graph, ZeroPortal, ZeroBanner, JsonConfig to master .. Updated Graph, ZeroPortal, ZeroBanner, JsonConfig to master Change-Id: I2af7bfdfffc0d4a42ae53bd2ed908b8e4ea5b035 --- M extensions/Graph M extensions/JsonConfig M extensions/ZeroBanner M extensions/ZeroPortal 4 files changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/67/162667/1 diff --git a/extensions/Graph b/extensions/Graph index e3baf85..09b9eff 16 --- a/extensions/Graph +++ b/extensions/Graph -Subproject commit e3baf852e19ec401b02969a891cdcbcaa3761d07 +Subproject commit 09b9eff6fcbf5da62d1e24592b75df2ada408cfe diff --git a/extensions/JsonConfig b/extensions/JsonConfig index ef47b98..e9462bf 16 --- a/extensions/JsonConfig +++ b/extensions/JsonConfig -Subproject commit ef47b9851149dffc88f21b187412ea9411645f3b +Subproject commit e9462bfd58a18f60414bb2c34ecdc91a5a674cc4 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index 716abfb..d5aa4e8 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit 716abfbe916852ed85c105ba0f819e030430c46b +Subproject commit d5aa4e8430c91c69ae5e14085eafcafe49cb24f8 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 493fba7..1500beb 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 493fba73e518e70c6ce5a070acbba83a73e703f3 +Subproject commit 1500beb5c95702c58138780f2f09c6e5a4491b1c -- To view, visit https://gerrit.wikimedia.org/r/162667 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I2af7bfdfffc0d4a42ae53bd2ed908b8e4ea5b035 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.24wmf22 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated Graph, ZeroPortal, ZeroBanner, JsonConfig to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/162668 Change subject: Updated Graph, ZeroPortal, ZeroBanner, JsonConfig to master .. Updated Graph, ZeroPortal, ZeroBanner, JsonConfig to master Change-Id: I18a624e50e19adc91558ffdc50e3713036f42a7c --- M extensions/Graph M extensions/JsonConfig M extensions/ZeroBanner M extensions/ZeroPortal 4 files changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/68/162668/1 diff --git a/extensions/Graph b/extensions/Graph index 922aac1..09b9eff 16 --- a/extensions/Graph +++ b/extensions/Graph -Subproject commit 922aac1b6ed21b6808be049c9dbb63cff8e97c7c +Subproject commit 09b9eff6fcbf5da62d1e24592b75df2ada408cfe diff --git a/extensions/JsonConfig b/extensions/JsonConfig index a3e9a2e..e9462bf 16 --- a/extensions/JsonConfig +++ b/extensions/JsonConfig -Subproject commit a3e9a2e53298efd4b6d47cabcd7b0d9a2584a616 +Subproject commit e9462bfd58a18f60414bb2c34ecdc91a5a674cc4 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index e0d1a06..d5aa4e8 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit e0d1a06465d7b74dcd0874da57224d33eee5069a +Subproject commit d5aa4e8430c91c69ae5e14085eafcafe49cb24f8 diff --git a/extensions/ZeroPortal b/extensions/ZeroPortal index 442a906..1500beb 16 --- a/extensions/ZeroPortal +++ b/extensions/ZeroPortal -Subproject commit 442a906e1d372dc8880e52b936078c04d905 +Subproject commit 1500beb5c95702c58138780f2f09c6e5a4491b1c -- To view, visit https://gerrit.wikimedia.org/r/162668 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I18a624e50e19adc91558ffdc50e3713036f42a7c Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.24wmf21 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated ZeroBanner to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/162692 Change subject: Updated ZeroBanner to master .. Updated ZeroBanner to master Change-Id: Ia336935f5d44906d72b647e07f935b81454008be --- M extensions/ZeroBanner 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/92/162692/1 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index d5aa4e8..f794a84 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit d5aa4e8430c91c69ae5e14085eafcafe49cb24f8 +Subproject commit f794a84d5f8979b49f8c714404f3f48e6b4403d0 -- To view, visit https://gerrit.wikimedia.org/r/162692 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ia336935f5d44906d72b647e07f935b81454008be Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.24wmf21 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Updated ZeroBanner to master - change (mediawiki/core)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/162693 Change subject: Updated ZeroBanner to master .. Updated ZeroBanner to master Change-Id: Ia4753f0dda5476878414b9f29d4ca9fef440ac02 --- M extensions/ZeroBanner 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/93/162693/1 diff --git a/extensions/ZeroBanner b/extensions/ZeroBanner index d5aa4e8..f794a84 16 --- a/extensions/ZeroBanner +++ b/extensions/ZeroBanner -Subproject commit d5aa4e8430c91c69ae5e14085eafcafe49cb24f8 +Subproject commit f794a84d5f8979b49f8c714404f3f48e6b4403d0 -- To view, visit https://gerrit.wikimedia.org/r/162693 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ia4753f0dda5476878414b9f29d4ca9fef440ac02 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.24wmf22 Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Moved 470-01 to unified, zero-only, https support - change (operations/puppet)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/162770
Change subject: Moved 470-01 to unified, zero-only, https support
..
Moved 470-01 to unified, zero-only, https support
Change-Id: Ie99cdf396d1a80771f203701fc7c39555ee60b74
---
M templates/varnish/zero.inc.vcl.erb
1 file changed, 9 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/70/162770/1
diff --git a/templates/varnish/zero.inc.vcl.erb
b/templates/varnish/zero.inc.vcl.erb
index 90cc5ef..96721d2 100644
--- a/templates/varnish/zero.inc.vcl.erb
+++ b/templates/varnish/zero.inc.vcl.erb
@@ -150,7 +150,7 @@
set req.http.X-CS = req.http.X-CS2;
}
}
-} else if (req.http.X-CS2 ~ ^(470-01|520-18)$) {
+} else if (req.http.X-CS2 ~ ^(520-18)$) {
if ((!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) ||
req.http.X-Forwarded-By == Opera) {
if (req.http.X-Subdomain == ZERO) {
// ZERO only, direct HTTP or opera
@@ -188,7 +188,7 @@
} else {
//
// Each carrier migrated to the unified design must be moved to
the vcl_deliver so that analytics stays current
-// Enabled:
250-99,401-01,404-01,414-06,416-03,426-04,428-98,429-02,436-01,436-04,437-01,514-02,515-03,635-10,TESTON
+// Enabled:
250-99,401-01,404-01,414-06,416-03,426-04,428-98,429-02,436-01,436-04,437-01,470-01,514-02,515-03,635-10,TESTON
// Disabled:
255-03,310-260,410-06,436-05,515-05,604-00,621-30,623-03,630-03,631-02
//
// TBD: Beware of legitimately formed links happening to have
:ZeroRatedMobileAccess...
@@ -270,6 +270,13 @@
if ((!req.http.X-Forwarded-Proto !req.http.X-Forwarded-By) ||
req.http.X-Forwarded-By == Opera) {
set req.http.X-CS = req.http.X-CS2;
}
+} else if (req.http.X-CS2 == 470-01) {
+// These carriers support HTTPS for DIRECT and OPERA, but ZERO-only
+if (!req.http.X-Forwarded-By || req.http.X-Forwarded-By ==
Opera) {
+if (req.http.X-Subdomain == ZERO) {
+set req.http.X-CS = req.http.X-CS2;
+}
+}
} else if (req.http.X-CS2 ~
^(255-03|310-260|410-06|436-05|515-05|604-00|621-30|623-03|630-03|631-02|TESTON)(\|.*)?$)
{
// Support HTTPS, allows all proxies
// Once analytics is fixed, this will be done for everything
--
To view, visit https://gerrit.wikimedia.org/r/162770
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie99cdf396d1a80771f203701fc7c39555ee60b74
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Minor code cleanup and a doc type cleanup - change (mediawiki/core)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/162812
Change subject: Minor code cleanup and a doc type cleanup
..
Minor code cleanup and a doc type cleanup
Change-Id: Ib89226a83738d1205405eac3926bfaa578966133
---
M includes/EditPage.php
M includes/Status.php
2 files changed, 3 insertions(+), 7 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/12/162812/1
diff --git a/includes/EditPage.php b/includes/EditPage.php
index a9925ff..8a55b1d 100644
--- a/includes/EditPage.php
+++ b/includes/EditPage.php
@@ -2230,11 +2230,7 @@
* $this-allowNonTextContent is not true.
*/
protected function toEditText( $content ) {
- if ( $content === null || $content === false ) {
- return $content;
- }
-
- if ( is_string( $content ) ) {
+ if ( $content === null || $content === false || is_string(
$content ) ) {
return $content;
}
diff --git a/includes/Status.php b/includes/Status.php
index 1a72968..2a9d173 100644
--- a/includes/Status.php
+++ b/includes/Status.php
@@ -184,9 +184,9 @@
/**
* Get the error list as a wikitext formatted list
*
-* @param string $shortContext A short enclosing context message name,
to
+* @param string|bool $shortContext A short enclosing context message
name, to
*be used when there is a single error
-* @param string $longContext A long enclosing context message name,
for a list
+* @param string|bool $longContext A long enclosing context message
name, for a list
* @return string
*/
public function getWikiText( $shortContext = false, $longContext =
false ) {
--
To view, visit https://gerrit.wikimedia.org/r/162812
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib89226a83738d1205405eac3926bfaa578966133
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Only hide images in zero-subdomain - change (mediawiki...ZeroBanner)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/162380
Change subject: Only hide images in zero-subdomain
..
Only hide images in zero-subdomain
Change-Id: If3d5e4ec90ff3790e544b3d7c86f97f9aad2958f
---
M includes/PageRendering.php
1 file changed, 1 insertion(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroBanner
refs/changes/80/162380/1
diff --git a/includes/PageRendering.php b/includes/PageRendering.php
index 01a2290..58f4bd0 100644
--- a/includes/PageRendering.php
+++ b/includes/PageRendering.php
@@ -340,7 +340,7 @@
* @return bool
*/
public function onSpecialMobileEditWatchlist_images( $images ) {
- if ( !$this-config-showImages() ) {
+ if ( $this-isZeroSubdomain() !$this-config-showImages() )
{
$images = array();
return false;
}
--
To view, visit https://gerrit.wikimedia.org/r/162380
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If3d5e4ec90ff3790e544b3d7c86f97f9aad2958f
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/ZeroBanner
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Ignore VIA before 2014-03-22 (wasn't in X-Analytics) - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/161804
Change subject: Ignore VIA before 2014-03-22 (wasn't in X-Analytics)
..
Ignore VIA before 2014-03-22 (wasn't in X-Analytics)
Change-Id: I035f2598d315bd4f19d46660b9a966d326469939
---
M scripts/weblogs.py
1 file changed, 5 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/04/161804/1
diff --git a/scripts/weblogs.py b/scripts/weblogs.py
index ab5d3f1..1d0a136 100644
--- a/scripts/weblogs.py
+++ b/scripts/weblogs.py
@@ -216,6 +216,8 @@
def combineStats(self):
safePrint('Combine stat files')
+# Logs did not contain the VIA X-Analytics tag before this date
+ignoreViaBefore = datetime(2014, 3, 22)
configs = self.downloadConfigs()
stats = collections.defaultdict(int)
for f in os.listdir(self.pathCache):
@@ -251,7 +253,7 @@
(conf.https or https == u'http') and \
(True == langs or lang in langs) and \
(True == sites or site2 in sites) and \
-(via in conf.via) and \
+(dt ignoreViaBefore or via in conf.via)
and \
(ipset in conf.ipsets):
isZero = True
break
@@ -351,5 +353,5 @@
if __name__ == '__main__':
-WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
-# WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
+# WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
+WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
--
To view, visit https://gerrit.wikimedia.org/r/161804
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I035f2598d315bd4f19d46660b9a966d326469939
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Ignore VIA before 2014-03-22 (wasn't in X-Analytics) - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: Ignore VIA before 2014-03-22 (wasn't in X-Analytics)
..
Ignore VIA before 2014-03-22 (wasn't in X-Analytics)
Change-Id: I035f2598d315bd4f19d46660b9a966d326469939
---
M scripts/weblogs.py
1 file changed, 5 insertions(+), 3 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/weblogs.py b/scripts/weblogs.py
index ab5d3f1..1d0a136 100644
--- a/scripts/weblogs.py
+++ b/scripts/weblogs.py
@@ -216,6 +216,8 @@
def combineStats(self):
safePrint('Combine stat files')
+# Logs did not contain the VIA X-Analytics tag before this date
+ignoreViaBefore = datetime(2014, 3, 22)
configs = self.downloadConfigs()
stats = collections.defaultdict(int)
for f in os.listdir(self.pathCache):
@@ -251,7 +253,7 @@
(conf.https or https == u'http') and \
(True == langs or lang in langs) and \
(True == sites or site2 in sites) and \
-(via in conf.via) and \
+(dt ignoreViaBefore or via in conf.via)
and \
(ipset in conf.ipsets):
isZero = True
break
@@ -351,5 +353,5 @@
if __name__ == '__main__':
-WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
-# WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
+# WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).manualRun()
+WebLogProcessor(logDatePattern=(sys.argv[1] if len(sys.argv) 1 else
False)).safeRun()
--
To view, visit https://gerrit.wikimedia.org/r/161804
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I035f2598d315bd4f19d46660b9a966d326469939
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
Gerrit-Reviewer: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] filter out last day of data, simplify iszero-zero - change (analytics/zero-sms)
Yurik has submitted this change and it was merged. Change subject: filter out last day of data, simplify iszero-zero .. filter out last day of data, simplify iszero-zero Change-Id: If414ca8774a307f75402a4ffb6ad7aa5494307d3 --- M scripts/weblogs.py 1 file changed, 6 insertions(+), 3 deletions(-) Approvals: Yurik: Verified; Looks good to me, approved diff --git a/scripts/weblogs.py b/scripts/weblogs.py index 1d0a136..3fbd90c 100644 --- a/scripts/weblogs.py +++ b/scripts/weblogs.py @@ -257,7 +257,7 @@ (ipset in conf.ipsets): isZero = True break -vals[9] = u'INCL' if isZero else u'EXCL' +vals[9] = u'yes' if isZero else u'no' else: vals[9] = '' else: @@ -284,14 +284,17 @@ else: df = DataFrame(stats, columns=columnHeaders11) +# filter type==DATA data = df[df['type'] == 'DATA'] -data['iszero'] = data['zero'].map(lambda v: 'yes' if v == 'INCL' else 'no') +# filter out last date +lastDate = data.date.max() +data = data[data.date lastDate] xcs = list(data.xcs.unique()) for id in xcs: s = StringIO.StringIO() -pivot_table(data[data.xcs == id], 'count', ['date', 'iszero'], aggfunc=np.sum).to_csv(s, header=True) +pivot_table(data[data.xcs == id], 'count', ['date', 'zero'], aggfunc=np.sum).to_csv(s, header=True) result = s.getvalue() # sortColumns = ['date', 'via', 'ipset', 'https', 'lang', 'subdomain', 'site', 'zero'] -- To view, visit https://gerrit.wikimedia.org/r/161806 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: If414ca8774a307f75402a4ffb6ad7aa5494307d3 Gerrit-PatchSet: 1 Gerrit-Project: analytics/zero-sms Gerrit-Branch: master Gerrit-Owner: Yurik yu...@wikimedia.org Gerrit-Reviewer: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] filter out last day of data, simplify iszero-zero - change (analytics/zero-sms)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/161806 Change subject: filter out last day of data, simplify iszero-zero .. filter out last day of data, simplify iszero-zero Change-Id: If414ca8774a307f75402a4ffb6ad7aa5494307d3 --- M scripts/weblogs.py 1 file changed, 6 insertions(+), 3 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms refs/changes/06/161806/1 diff --git a/scripts/weblogs.py b/scripts/weblogs.py index 1d0a136..3fbd90c 100644 --- a/scripts/weblogs.py +++ b/scripts/weblogs.py @@ -257,7 +257,7 @@ (ipset in conf.ipsets): isZero = True break -vals[9] = u'INCL' if isZero else u'EXCL' +vals[9] = u'yes' if isZero else u'no' else: vals[9] = '' else: @@ -284,14 +284,17 @@ else: df = DataFrame(stats, columns=columnHeaders11) +# filter type==DATA data = df[df['type'] == 'DATA'] -data['iszero'] = data['zero'].map(lambda v: 'yes' if v == 'INCL' else 'no') +# filter out last date +lastDate = data.date.max() +data = data[data.date lastDate] xcs = list(data.xcs.unique()) for id in xcs: s = StringIO.StringIO() -pivot_table(data[data.xcs == id], 'count', ['date', 'iszero'], aggfunc=np.sum).to_csv(s, header=True) +pivot_table(data[data.xcs == id], 'count', ['date', 'zero'], aggfunc=np.sum).to_csv(s, header=True) result = s.getvalue() # sortColumns = ['date', 'via', 'ipset', 'https', 'lang', 'subdomain', 'site', 'zero'] -- To view, visit https://gerrit.wikimedia.org/r/161806 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: If414ca8774a307f75402a4ffb6ad7aa5494307d3 Gerrit-PatchSet: 1 Gerrit-Project: analytics/zero-sms Gerrit-Branch: master Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Renamed zero fld to iszero (val=yes/no) - change (analytics/zero-sms)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/161808
Change subject: Renamed zero fld to iszero (val=yes/no)
..
Renamed zero fld to iszero (val=yes/no)
Change-Id: I233231e5043649a973d963b5a91d96159b9a0a4f
---
M scripts/weblogs.py
1 file changed, 4 insertions(+), 4 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/08/161808/1
diff --git a/scripts/weblogs.py b/scripts/weblogs.py
index 3fbd90c..90522a4 100644
--- a/scripts/weblogs.py
+++ b/scripts/weblogs.py
@@ -22,7 +22,7 @@
columnHeaders10 =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
-columnHeaders11 =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,zero,count'.split(',')
+columnHeaders11 =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,iszero,count'.split(',')
validSubDomains = {'m', 'zero', 'mobile', 'wap'}
validHttpCode = {'200', '304'}
@@ -294,11 +294,11 @@
for id in xcs:
s = StringIO.StringIO()
-pivot_table(data[data.xcs == id], 'count', ['date', 'zero'],
aggfunc=np.sum).to_csv(s, header=True)
+pivot_table(data[data.xcs == id], 'count', ['date', 'iszero'],
aggfunc=np.sum).to_csv(s, header=True)
result = s.getvalue()
-# sortColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'zero']
-# outColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'zero', 'count']
+# sortColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'iszero']
+# outColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'iszero', 'count']
# xcsData = data[data.xcs == id].sort(columns=sortColumns)
# result =
xcsData.sort(columns=sortColumns).to_csv(columns=outColumns, index=False)
--
To view, visit https://gerrit.wikimedia.org/r/161808
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I233231e5043649a973d963b5a91d96159b9a0a4f
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Renamed zero fld to iszero (val=yes/no) - change (analytics/zero-sms)
Yurik has submitted this change and it was merged.
Change subject: Renamed zero fld to iszero (val=yes/no)
..
Renamed zero fld to iszero (val=yes/no)
Change-Id: I233231e5043649a973d963b5a91d96159b9a0a4f
---
M scripts/weblogs.py
1 file changed, 4 insertions(+), 4 deletions(-)
Approvals:
Yurik: Verified; Looks good to me, approved
diff --git a/scripts/weblogs.py b/scripts/weblogs.py
index 3fbd90c..90522a4 100644
--- a/scripts/weblogs.py
+++ b/scripts/weblogs.py
@@ -22,7 +22,7 @@
columnHeaders10 =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,count'.split(',')
-columnHeaders11 =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,zero,count'.split(',')
+columnHeaders11 =
u'date,type,xcs,via,ipset,https,lang,subdomain,site,iszero,count'.split(',')
validSubDomains = {'m', 'zero', 'mobile', 'wap'}
validHttpCode = {'200', '304'}
@@ -294,11 +294,11 @@
for id in xcs:
s = StringIO.StringIO()
-pivot_table(data[data.xcs == id], 'count', ['date', 'zero'],
aggfunc=np.sum).to_csv(s, header=True)
+pivot_table(data[data.xcs == id], 'count', ['date', 'iszero'],
aggfunc=np.sum).to_csv(s, header=True)
result = s.getvalue()
-# sortColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'zero']
-# outColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'zero', 'count']
+# sortColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'iszero']
+# outColumns = ['date', 'via', 'ipset', 'https', 'lang',
'subdomain', 'site', 'iszero', 'count']
# xcsData = data[data.xcs == id].sort(columns=sortColumns)
# result =
xcsData.sort(columns=sortColumns).to_csv(columns=outColumns, index=False)
--
To view, visit https://gerrit.wikimedia.org/r/161808
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I233231e5043649a973d963b5a91d96159b9a0a4f
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
Gerrit-Reviewer: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Keep debug flag while editing a page - change (mediawiki/core)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/161878
Change subject: Keep debug flag while editing a page
..
Keep debug flag while editing a page
If request contained 'debug' field, keep it around
for the subsequent edit requests as a hidden field.
Change-Id: I6d401ce66fe5b3e131c1673b9d665fb9899228c3
---
M includes/EditPage.php
1 file changed, 6 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/78/161878/1
diff --git a/includes/EditPage.php b/includes/EditPage.php
index a9925ff..421ad86 100644
--- a/includes/EditPage.php
+++ b/includes/EditPage.php
@@ -2284,7 +2284,8 @@
* during form output near the top, for captchas and the like.
*/
function showEditForm( $formCallback = null ) {
- global $wgOut, $wgUser;
+ global $wgOut, $wgUser, $wgRequest;
+;
wfProfileIn( __METHOD__ );
@@ -2422,6 +2423,10 @@
$wgOut-addHTML( Html::hidden( 'format', $this-contentFormat )
);
$wgOut-addHTML( Html::hidden( 'model', $this-contentModel ) );
+ if ( $wgRequest-getCheck( 'debug' ) ) {
+ $wgOut-addHTML( Html::hidden( 'debug', '1' ) );
+ }
+
if ( $this-section == 'new' ) {
$this-showSummaryInput( true, $this-summary );
$wgOut-addHTML( $this-getSummaryPreview( true,
$this-summary ) );
--
To view, visit https://gerrit.wikimedia.org/r/161878
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I6d401ce66fe5b3e131c1673b9d665fb9899228c3
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] new JCSingleton::getContentClass($modelId) - change (mediawiki...JsonConfig)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/161893
Change subject: new JCSingleton::getContentClass($modelId)
..
new JCSingleton::getContentClass($modelId)
Change-Id: I2ea5ce57059c26058dd92f3941c02ef551088159
---
M includes/JCContentHandler.php
M includes/JCSingleton.php
2 files changed, 23 insertions(+), 19 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/JsonConfig
refs/changes/93/161893/1
diff --git a/includes/JCContentHandler.php b/includes/JCContentHandler.php
index b4b077e..40a5d90 100644
--- a/includes/JCContentHandler.php
+++ b/includes/JCContentHandler.php
@@ -52,27 +52,10 @@
* @return JCContent the JsonSchemaContent object wrapping $text
*/
public function unserializeContent( $text, $format = null, $isSaving =
true ) {
- global $wgJsonConfigModels;
$this-checkFormat( $format );
$modelId = $this-getModelID();
- $class = null;
- if ( array_key_exists( $modelId, $wgJsonConfigModels ) ) {
- $value = $wgJsonConfigModels[$modelId];
- if ( is_array( $value ) ) {
- if ( !array_key_exists( 'class', $value ) ) {
- wfLogWarning( JsonConfig: Invalid
\$wgJsonConfigModels['$modelId'] array value, 'class' not found );
- } else {
- $class = $value['class'];
- }
- } else {
- $class = $value;
- }
- }
- if ( $class ) {
- return new $class( $text, $modelId, $isSaving );
- } else {
- return new JCContent( $text, $modelId, $isSaving );
- }
+ $class = JCSingleton::getContentClass( $modelId );
+ return new $class( $text, $modelId, $isSaving );
}
/**
diff --git a/includes/JCSingleton.php b/includes/JCSingleton.php
index 46aa44e..de69b05 100644
--- a/includes/JCSingleton.php
+++ b/includes/JCSingleton.php
@@ -336,6 +336,27 @@
return self::$titleMap;
}
+ public static function getContentClass( $modelId ) {
+ global $wgJsonConfigModels;
+ $class = null;
+ if ( array_key_exists( $modelId, $wgJsonConfigModels ) ) {
+ $value = $wgJsonConfigModels[$modelId];
+ if ( is_array( $value ) ) {
+ if ( !array_key_exists( 'class', $value ) ) {
+ wfLogWarning( JsonConfig: Invalid
\$wgJsonConfigModels['$modelId'] array value, 'class' not found );
+ } else {
+ $class = $value['class'];
+ }
+ } else {
+ $class = $value;
+ }
+ }
+ if ( !$class ) {
+ $class = __NAMESPACE__ . '\JCContent';
+ }
+ return $class;
+ }
+
/**
* Returns an array with settings if the $titleValue object is handled
by the JsonConfig extension,
* false if unrecognized namespace, and null if namespace is handled
but not this title
--
To view, visit https://gerrit.wikimedia.org/r/161893
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2ea5ce57059c26058dd92f3941c02ef551088159
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/JsonConfig
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Auto-refresh graph on editing - change (mediawiki...Graph)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/161894
Change subject: Auto-refresh graph on editing
..
Auto-refresh graph on editing
Requires https://gerrit.wikimedia.org/r/161893
Change-Id: I12f481c10576ffe53f60d820900dc0dd876902a1
---
M Graph.body.php
M Graph.php
A js/graph.editor.js
M js/graph.js
4 files changed, 65 insertions(+), 46 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Graph
refs/changes/94/161894/1
diff --git a/Graph.body.php b/Graph.body.php
index 867ad71..9b89931 100644
--- a/Graph.body.php
+++ b/Graph.body.php
@@ -13,6 +13,7 @@
use Html;
use JsonConfig\JCContent;
use JsonConfig\JCContentView;
+use JsonConfig\JCSingleton;
use Parser;
use ParserOptions;
use ParserOutput;
@@ -55,6 +56,21 @@
$parserOutput-addModules( 'ext.graph' );
return $parserOutput;
}
+
+ /**
+* @param \EditPage $editpage
+* @param \OutputPage $output
+* @return bool
+*/
+ public static function editPageShowEditFormInitial( $editpage, $output
) {
+ // TODO: not sure if this is the best way to test
+ if ( $editpage-contentFormat === CONTENT_FORMAT_JSON
+JCSingleton::getContentClass( $editpage-contentModel )
=== __NAMESPACE__ . '\Content'
+ ) {
+ $output-addModules( 'ext.graph.editor' );
+ }
+ return true;
+ }
}
/**
diff --git a/Graph.php b/Graph.php
index 34a3ec6..81aa27e 100644
--- a/Graph.php
+++ b/Graph.php
@@ -47,49 +47,17 @@
$wgGraphDataDomains = array();
$wgHooks['ParserFirstCallInit'][] = 'Graph\Singleton::onParserFirstCallInit';
+$wgHooks['EditPage::showEditForm:initial'][] =
'Graph\Singleton::editPageShowEditFormInitial';
-
-// ResourceLoader modules
-/**
- * A boilerplate for resource loader modules
- */
$extGraphBoilerplate = array(
'localBasePath' = __DIR__,
'remoteExtPath' = 'Graph',
'targets' = array( 'mobile', 'desktop' ),
);
-$wgResourceModules['mediawiki.libs.d3'] = array(
-'scripts' = array(
-'lib/d3.js',
-//'lib/d3.geo.projection.min.js',
-),
-) + $extGraphBoilerplate;
-$wgResourceModules['mediawiki.libs.topojson'] = array(
-'scripts' = array(
-'lib/topojson.js',
-),
-) + $extGraphBoilerplate;
-$wgResourceModules['mediawiki.libs.vega'] = array(
-'dependencies' = array(
-'mediawiki.libs.d3',
-'mediawiki.libs.topojson',
-),
-'scripts' = array(
-'lib/vega.js',
-),
-) + $extGraphBoilerplate;
$wgResourceModules['ext.graph'] = array(
-// TODO: dependencies don't work. Symptoms:
-// * Firefox works
-// * Chrome works in debug mode
-// * Chrome does not work in production mode (debug=false)
-//'dependencies' = array(
-//'mediawiki.libs.vega',
-//),
'scripts' = array(
'lib/d3.js',
-// 'lib/d3.geo.projection.min.js',
'lib/topojson.js',
'lib/vega.js',
'js/graph.js',
@@ -98,4 +66,11 @@
'styles/common.less',
),
) + $extGraphBoilerplate;
+
+$wgResourceModules['ext.graph.editor'] = array(
+'scripts' = array(
+'js/graph.editor.js',
+)
+) + $extGraphBoilerplate;
+
unset( $extGraphBoilerplate );
diff --git a/js/graph.editor.js b/js/graph.editor.js
new file mode 100644
index 000..7670cc4
--- /dev/null
+++ b/js/graph.editor.js
@@ -0,0 +1,30 @@
+( function( $ ) {
+ var oldContent = '';
+ (function() {
+ $('.mw-wiki-graph').each(function () {
+ // This should not be a loop - there must be only one
value with this class
+ var el = this,
+ textbox = $('#wpTextbox1'),
+ context = textbox
textbox.data('wikiEditor-context');
+
+ if (!context || !context.evt ||
!context.evt.codeEditorSync || !context.fn) {
+ return;
+ }
+ context.evt.codeEditorSync();
+ var content = context.fn.getContents();
+ if (oldContent === content) {
+ return;
+ }
+ oldContent = content;
+ var annotations =
context.codeEditor.getSession().getAnnotations();
+ if (annotations.every(function(v) { return v.type !==
'error'; })) {
+ if (this.hasAttribute('spec'))
+ this.removeAttribute('spec');
+ var spec = $.parseJSON(content);
+ vg.parse.spec(spec, function(chart) {
chart({el:el}).update(); })
+ }
+ });
+ // FIXME: This should be
[MediaWiki-commits] [Gerrit] Pass CampaignContent as a constructor param - change (mediawiki...UploadWizard)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/161895
Change subject: Pass CampaignContent as a constructor param
..
Pass CampaignContent as a constructor param
Change-Id: Ibabe3a10369863a0cb53115d97f31ea53583c160
---
M includes/CampaignContentHandler.php
1 file changed, 1 insertion(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/UploadWizard
refs/changes/95/161895/1
diff --git a/includes/CampaignContentHandler.php
b/includes/CampaignContentHandler.php
index 2097e55..6e1de6e 100644
--- a/includes/CampaignContentHandler.php
+++ b/includes/CampaignContentHandler.php
@@ -13,10 +13,6 @@
class CampaignContentHandler extends JsonContentHandler {
public function __construct( $modelId = 'Campaign' ) {
- parent::__construct( $modelId );
- }
-
- protected function getContentClass() {
- return 'CampaignContent';
+ parent::__construct( $modelId, 'CampaignContent' );
}
}
--
To view, visit https://gerrit.wikimedia.org/r/161895
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibabe3a10369863a0cb53115d97f31ea53583c160
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/UploadWizard
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Declared missing var, fixed php doc spelling, spacing - change (mediawiki...WikiEditor)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/161896
Change subject: Declared missing var, fixed php doc spelling, spacing
..
Declared missing var, fixed php doc spelling, spacing
Change-Id: I7a60eeb224cc0e1c013f59df42fc4a7805df8a5b
---
M WikiEditor.hooks.php
1 file changed, 8 insertions(+), 7 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/WikiEditor
refs/changes/96/161896/1
diff --git a/WikiEditor.hooks.php b/WikiEditor.hooks.php
index 8c6800c..9226baa 100644
--- a/WikiEditor.hooks.php
+++ b/WikiEditor.hooks.php
@@ -155,8 +155,8 @@
*
* Adds the modules to the edit form
*
-* @param $editPage EditPage the current EditPage object.
-* @param $output OutputPage object.
+* @param EditPage $editPage the current EditPage object.
+* @param OutputPage $outputPage object.
* @return bool
*/
public static function editPageShowEditFormInitial( $editPage,
$outputPage ) {
@@ -202,10 +202,10 @@
/**
* GetPreferences hook
*
-* Adds WikiEditor-releated items to the preferences
+* Adds WikiEditor-related items to the preferences
*
-* @param $user User current user
-* @param $defaultPreferences array list of default user preference
controls
+* @param User $user current user
+* @param array $defaultPreferences list of default user preference
controls
* @return bool
*/
public static function getPreferences( $user, $defaultPreferences ) {
@@ -285,9 +285,10 @@
'img_framed',
'img_frameless',
);
+ $magicWords = array();
foreach ( $requiredMagicWords as $name ) {
- $magicWords[$name] = MagicWord::get( $name
)-getSynonym( 0 );
- }
+ $magicWords[$name] = MagicWord::get( $name
)-getSynonym( 0 );
+ }
$vars['wgWikiEditorMagicWords'] = $magicWords;
}
--
To view, visit https://gerrit.wikimedia.org/r/161896
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7a60eeb224cc0e1c013f59df42fc4a7805df8a5b
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/WikiEditor
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Use single quotes for pre (easier to read) - change (mediawiki...Scribunto)
Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/161897 Change subject: Use single quotes for pre (easier to read) .. Use single quotes for pre (easier to read) Change-Id: Ib666f56471fc8ea7dac36367c962788d1a304100 --- M common/ScribuntoContent.php 1 file changed, 1 insertion(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Scribunto refs/changes/97/161897/1 diff --git a/common/ScribuntoContent.php b/common/ScribuntoContent.php index 920db00..a7d1558 100644 --- a/common/ScribuntoContent.php +++ b/common/ScribuntoContent.php @@ -146,7 +146,7 @@ // No GeSHi, or GeSHi can't parse it, use plain pre $output-setText( $output-getText() . - pre class=\mw-code mw-script\ dir=\ltr\\n . + pre class='mw-code mw-script' dir='ltr'\n . htmlspecialchars( $text ) . \n/pre\n ); -- To view, visit https://gerrit.wikimedia.org/r/161897 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ib666f56471fc8ea7dac36367c962788d1a304100 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/Scribunto Gerrit-Branch: master Gerrit-Owner: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Use constructor param for the content class - change (mediawiki...MassMessage)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/161898
Change subject: Use constructor param for the content class
..
Use constructor param for the content class
Change-Id: I5746fa3af70837858bffd10da9b5165fb1a5a10e
---
M includes/content/MassMessageListContentHandler.php
1 file changed, 1 insertion(+), 8 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/MassMessage
refs/changes/98/161898/1
diff --git a/includes/content/MassMessageListContentHandler.php
b/includes/content/MassMessageListContentHandler.php
index c2eadd2..c12aaa0 100644
--- a/includes/content/MassMessageListContentHandler.php
+++ b/includes/content/MassMessageListContentHandler.php
@@ -6,7 +6,7 @@
* @param string $modelId
*/
public function __construct( $modelId = 'MassMessageListContent' ) {
- parent::__construct( $modelId );
+ parent::__construct( $modelId, 'MassMessageListContent' );
}
/**
@@ -29,13 +29,6 @@
*/
public function makeEmptyContent() {
return new MassMessageListContent(
'{description:,targets:[]}' );
- }
-
- /**
-* @return string
-*/
- protected function getContentClass() {
- return 'MassMessageListContent';
}
/**
--
To view, visit https://gerrit.wikimedia.org/r/161898
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5746fa3af70837858bffd10da9b5165fb1a5a10e
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/MassMessage
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Don't rely on parent object table, use ctor param for conten... - change (mediawiki...EventLogging)
Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/161899
Change subject: Don't rely on parent object table, use ctor param for content
class
..
Don't rely on parent object table, use ctor param for content class
Change-Id: I124259b701f6d550d8595238bd32180f585640f2
---
M includes/JsonSchemaContent.php
M includes/JsonSchemaContentHandler.php
2 files changed, 17 insertions(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/EventLogging
refs/changes/99/161899/1
diff --git a/includes/JsonSchemaContent.php b/includes/JsonSchemaContent.php
index 96e5709..91247c0 100644
--- a/includes/JsonSchemaContent.php
+++ b/includes/JsonSchemaContent.php
@@ -75,6 +75,22 @@
}
/**
+* Constructs an HTML representation of a JSON object.
+* @param array $mapping
+* @return string HTML
+*/
+ protected function objectTable( $mapping ) {
+ $rows = array();
+
+ foreach ( $mapping as $key = $val ) {
+ $rows[] = $this-objectRow( $key, $val );
+ }
+ return Xml::tags( 'table', array( 'class' = 'mw-json' ),
+ Xml::tags( 'tbody', array(), join( \n, $rows ) )
+ );
+ }
+
+ /**
* Constructs HTML representation of a single key-value pair.
* Override this to support $ref
* @param string $key
diff --git a/includes/JsonSchemaContentHandler.php
b/includes/JsonSchemaContentHandler.php
index 25c6c84..e0943c9 100644
--- a/includes/JsonSchemaContentHandler.php
+++ b/includes/JsonSchemaContentHandler.php
@@ -12,10 +12,6 @@
class JsonSchemaContentHandler extends JsonContentHandler {
public function __construct( $modelId = 'JsonSchema' ) {
- parent::__construct( $modelId, array( CONTENT_FORMAT_JSON ) );
- }
-
- protected function getContentClass() {
- return 'JsonSchemaContent';
+ parent::__construct( $modelId, 'JsonSchemaContent' );
}
}
--
To view, visit https://gerrit.wikimedia.org/r/161899
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I124259b701f6d550d8595238bd32180f585640f2
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/EventLogging
Gerrit-Branch: master
Gerrit-Owner: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
