Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/172661
Change subject: Updated weblogs2 to create single pages
......................................................................
Updated weblogs2 to create single pages
Change-Id: Ic2789a8478b01ee3117385f5e44eaf383c0db9f3
---
M scripts/run-hivezero.sh
M scripts/weblogs2.py
2 files changed, 48 insertions(+), 80 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/61/172661/1
diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh
index 2c8f95c..7688adc 100755
--- a/scripts/run-hivezero.sh
+++ b/scripts/run-hivezero.sh
@@ -31,7 +31,7 @@
if [ "$( date -d "$date" +%F 2>&1 | grep invalid )" = "" ] ; then
if [[ "$table" == 'wmf_raw.webrequest' ]]; then
-
path="/mnt/hdfs/wmf/data/raw/webrequest/webrequest_upload/hourly/$year/$month/$day/23"
+
path="/mnt/hdfs/wmf/data/raw/webrequest/webrequest_upload/hourly/$year/$(printf
"%02d" $month)/$(printf "%02d" $day)/23"
else
path="/mnt/hdfs/user/hive/warehouse/yurik.db/$table/year=$year/month=$month/day=$day"
fi
@@ -45,12 +45,14 @@
continue
fi
- if [[ "$6" -eq "overwrite" ]]; then
- hive -e "use yurik; ALTER TABLE zero_webstats DROP IF
EXISTS PARTITION(date = '$date');"
- else
-
path="/mnt/hdfs/user/hive/warehouse/yurik.db/zero_webstats/date="$date
- echo "***** Checking if '$path' exists"
- if [ -d $path ]; then
+
path="/mnt/hdfs/user/hive/warehouse/yurik.db/zero_webstats/date="$date
+ echo "***** Checking if '$path' exists"
+ if [ -d $path ]; then
+ if [ "$6" == "overwrite" ]; then
+ echo "***** Droping partition '$date'"
+ hive -e "use yurik; ALTER TABLE zero_webstats
DROP IF EXISTS PARTITION(date = '$date');"
+ else
+ echo "***** Skipping '$date'"
continue
fi
fi
diff --git a/scripts/weblogs2.py b/scripts/weblogs2.py
index 1e79027..1e0f727 100644
--- a/scripts/weblogs2.py
+++ b/scripts/weblogs2.py
@@ -4,7 +4,7 @@
import collections
from pandas import read_table, pivot_table
-from pandas.core.frame import DataFrame, Series
+from pandas.core.frame import DataFrame
import numpy as np
from logprocessor import *
@@ -58,13 +58,13 @@
ignoreViaBefore = datetime(2014, 3, 22)
configs = self.downloadConfigs()
stats = collections.defaultdict(int)
- for dateDir in os.listdir(self.pathCache):
+ for dateDir in os.listdir(self.pathLogs):
m = self.dateDirRe.match(dateDir)
if not m:
continue
dateStr = m.group(1)
dt = datetime.strptime(dateStr, '%Y-%m-%d')
- datePath = os.path.join(self.pathCache, dateDir)
+ datePath = os.path.join(self.pathLogs, dateDir)
for f in os.listdir(datePath):
if not self.fileRe.match(f):
continue
@@ -136,90 +136,56 @@
xcsList = [xcs for xcs in allData.xcs.unique() if xcs != 'ERROR' and
xcs[0:4] != 'TEST']
# filter type==DATA and site==wikipedia
- allData = allData[(allData['xcs'].isin(xcsList)) & (allData['site'] ==
'wikipedia')]
- # filter out last date
- lastDate = allData.date.max()
- df = allData[allData.date < lastDate]
+ df = allData[(allData['xcs'].isin(xcsList)) & (allData['site'] ==
'wikipedia')]
- allowedSubdomains = ['m', 'zero']
- limnCompat = df[(df.ison == 'y') & (df.iszero == 'y') &
(df.subdomain.isin(allowedSubdomains))]
s = StringIO.StringIO()
- pivot_table(limnCompat, 'count', ['date', 'xcs', 'subdomain'],
aggfunc=np.sum).to_csv(s, header=True)
- result = s.getvalue()
+ allowedSubdomains = ['m', 'zero']
+ dailySubdomains = df[(df.ison == 'y') & (df.iszero == 'y') &
(df.subdomain.isin(allowedSubdomains))]
+ pivot_table(dailySubdomains, 'count', ['date', 'xcs', 'subdomain'],
aggfunc=np.sum).to_csv(s, header=False)
wiki(
'edit',
title='RawData:DailySubdomains',
summary='refreshing data',
- text=result,
+ text='date,xcs,subdomain,count\n' + s.getvalue(),
token=wiki.token()
)
- # allEnabled = df[(df.ison == 'y') & (df.iszero == 'y')]
- # s = StringIO.StringIO()
- # pivot_table(allEnabled, 'count', ['date', 'xcs'],
aggfunc=np.sum).to_csv(s, header=True)
- # result = s.getvalue()
- #
- # wiki(
- # 'edit',
- # title='RawData:AllEnabled',
- # summary='refreshing data',
- # text=result,
- # token=wiki.token()
- # )
- return
- xcsList = list(df.xcs.unique())
- xcsList.sort()
- for id in xcsList:
- xcsDf = df[df.xcs == id]
- # create an artificial yes/opera value
- opera = xcsDf[(xcsDf.via == 'OPERA') & (xcsDf.iszero == 'yes')]
- opera['str'] = 'zero-opera'
+ # create an artificial yes/no/opera sums
+ opera = df[(df.via == 'OPERA') & (df.iszero == 'y')]
+ opera['str'] = 'o'
+ yes = df[df.iszero == 'y']
+ yes['str'] = 'y'
+ no = df[df.iszero == 'n']
+ no['str'] = 'n'
+ combined = opera.append(yes).append(no)
+ s = StringIO.StringIO()
+ pivot_table(combined, 'count', ['date', 'xcs', 'str'],
aggfunc=np.sum).to_csv(s, header=False)
- yes = xcsDf[xcsDf.iszero == 'yes']
- yes['str'] = 'zero-all'
+ wiki(
+ 'edit',
+ title='RawData:DailyTotals',
+ summary='refreshing data',
+ text='date,xcs,iszero,count\n' + s.getvalue(),
+ token=wiki.token()
+ )
- no = xcsDf[xcsDf.iszero == 'no']
- no['str'] = 'non-zero'
-
- combined = opera.append(yes).append(no)
-
- s = StringIO.StringIO()
- pivot_table(combined, 'count', ['date', 'str'],
aggfunc=np.sum).to_csv(s, header=False)
- result = 'date,iszero,count\n' + s.getvalue()
-
- wiki(
- 'edit',
- title='RawData:' + id,
- summary='refreshing data',
- text=result,
- token=wiki.token()
- )
-
- byLang = pivot_table(xcsDf, 'count', ['lang'],
aggfunc=np.sum).order('count', ascending=False)
+ results = ['lang,xcs,count']
+ for id in list(df.xcs.unique()):
+ byLang = pivot_table(df[df.xcs == id], 'count', ['lang'],
aggfunc=np.sum).order('count', ascending=False)
top = byLang.head(5)
- other = byLang.sum() - top.sum()
- s = StringIO.StringIO()
- Series.to_csv(top, s)
- result = 'lang,count\n' + s.getvalue() + ('other,%d\n' % other)
+ vals = list(top.iteritems())
+ vals.append(('other', byLang.sum() - top.sum()))
+ valsTotal = sum([v[1] for v in vals]) / 100.0
+ results.extend(['%s,%s,%.1f' % (l, id, c / valsTotal) for l, c in
vals])
- wiki(
- 'edit',
- title='RawData:' + id + '-langTotal',
- summary='refreshing data',
- text=result,
- token=wiki.token()
- )
-
- # return df
- # pt = pivot_table(df, values='count', index=['date'],
columns=['xcs','subdomain'], aggfunc=np.sum).head(10)
- # writeData(os.path.join(self.pathCache, 'combined-errors.tsv'),
- # ifilter(lambda v: v[1] == 'ERR', stats),
- # columnHeaders11)
- # writeData(os.path.join(self.pathCache, 'combined-stats.tsv'),
- # ifilter(lambda v: v[1] == 'STAT', stats), columnHeaders11)
- # writeData(os.path.join(self.pathCache, 'combined-data.tsv'),
- # ifilter(lambda v: v[1] == 'DATA', stats), columnHeaders11)
+ wiki(
+ 'edit',
+ title='RawData:LangPercent',
+ summary='refreshing data',
+ text='\n'.join(results),
+ token=wiki.token()
+ )
def run(self):
stats = self.combineStats()
--
To view, visit https://gerrit.wikimedia.org/r/172661
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic2789a8478b01ee3117385f5e44eaf383c0db9f3
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
