Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/169750
Change subject: updated scripts (misc changes)
......................................................................
updated scripts (misc changes)
Change-Id: Ib4a75794769c4aaaf155718cb9f1151a1ef4266b
---
M scripts/log2dfs.py
M scripts/run-hivezero.sh
M scripts/zero-counts.hql
3 files changed, 49 insertions(+), 32 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/50/169750/1
diff --git a/scripts/log2dfs.py b/scripts/log2dfs.py
index a4d7b51..30aa8d5 100644
--- a/scripts/log2dfs.py
+++ b/scripts/log2dfs.py
@@ -61,7 +61,7 @@
'FAKE_CACHE_STATUS': '',
'TCP_HIT': 'hit',
'TCP_IMS_HIT': 'hit',
- 'TCP_DENIED': '',
+ 'TCP_DENIED': 'denied',
'TCP_REFRESH_MISS': 'miss',
'TCP_NEGATIVE_HIT': 'hit',
}
@@ -76,6 +76,7 @@
self.logFileRe = re.compile(unicode(filePattern), re.IGNORECASE)
self.dateRe = re.compile(r'(201\d-\d\d-\d\dT\d\d):\d\d:\d\d(\.\d+)?')
self.urlRe = re.compile(r'^(https?)://([^/]+)([^?#]*)(.*)',
re.IGNORECASE)
+ self.xcsRe = re.compile(r'^[0-9]+-[0-9]+$', re.IGNORECASE)
def processLogFiles(self):
@@ -89,8 +90,7 @@
if statFile.endswith('.gz'):
statFile = statFile[:-3]
- # if not os.path.exists(statFile):
- if True:
+ if not os.path.exists(statFile):
self.processLogFile(logFile, statFile)
def processLogFile(self, logFile, statFile):
@@ -133,26 +133,26 @@
else:
streamData = io.open(logFile, 'r', encoding='utf8',
errors='ignore')
- with io.open(statFile, 'w', encoding='utf8') as out:
+ tmpFile = statFile + '.tmp'
+ with io.open(tmpFile, 'w', encoding='utf8') as out:
for line in streamData:
count += 1
if count % 1000000 == 0:
safePrint('%d lines processed' % count)
- strip = line.strip('\n\r')
+ strip = line.strip('\n\r\x00')
if isTab:
l = strip.split('\t')
- if l[2].startswith('201'):
+ if len(l) > 2 and l[2].startswith('201'):
while len(l) > 16:
l[13] += ' ' + l[14]
del l[11]
else:
l = strip.split(' ')
# fix text/html; charset=UTF-8 into one field
- while len(l) >= 10 and l[10].endswith(';') and l[11] !=
'-' and not l[11].startswith('http'):
+ while len(l) > 11 and l[10].endswith(';') and l[11] != '-'
and not l[11].startswith('http'):
l[10] += ' ' + l[11]
del l[11]
- l[13] = unquote(l[13])
if len(l) == 14:
l.append(u'')
l.append(u'')
@@ -162,12 +162,14 @@
safePrint(u'Wrong parts count - %d parts\n%s' %
(partsCount, line))
continue
- l = ['' if v == '-' else v for v in l]
+ l = ['' if v == '-' else v.replace('\t', ' ') for v in l]
(hostname, sequence, dt, time_firstbyte, ip, status,
response_size, http_method, uri, unknown1,
content_type, referer, x_forwarded_for, user_agent,
accept_language, x_analytics) = l
# status -> cache_status, http_status
# uri -> uri_host, uri_path, uri_query
# new: webrequest_source, year, month, day, hour
+
+ user_agent = unquote(user_agent).replace('\t', ' ')
m = self.dateRe.match(dt)
if not m:
@@ -180,6 +182,9 @@
month = unicode(d.month)
day = unicode(d.day)
hour = unicode(d.hour)
+
+ if self.xcsRe.match(x_analytics):
+ x_analytics = 'zero=' + x_analytics
if 'zero=' not in x_analytics:
if defaultXcs:
@@ -207,19 +212,23 @@
continue
cache_status = httpStatuses[cache_status]
- m = self.urlRe.match(uri)
- if not m:
- safePrint(u'URL parsing failed: "%s"\n%s' % (uri, line))
- continue
- if m.group(1).lower() == u'https' and u'https=' not in
x_analytics:
- x_analytics += u'https=1'
- uri_host = m.group(2)
- if uri_host.endswith(':80'):
- uri_host = uri_host[:-3]
- if uri_host.endswith('.'):
- uri_host = uri_host[:-1]
- uri_path = m.group(3)
- uri_query = m.group(4)
+ if uri == 'NONE://' or (http_method == 'CONNECT' and uri ==
':0'):
+ uri_host = uri_path = uri_query = ''
+
+ else:
+ m = self.urlRe.match(uri)
+ if not m:
+ safePrint(u'URL parsing failed: "%s"\n%s' % (uri,
line))
+ continue
+ if m.group(1).lower() == u'https' and u'https=' not in
x_analytics:
+ x_analytics += u'https=1'
+ uri_host = m.group(2)
+ if uri_host.endswith(':80'):
+ uri_host = uri_host[:-3]
+ if uri_host.endswith('.'):
+ uri_host = uri_host[:-1]
+ uri_path = m.group(3)
+ uri_query = m.group(4)
result = '\t'.join(
[hostname, sequence, dt, time_firstbyte, ip, cache_status,
http_status, response_size, http_method,
@@ -227,6 +236,11 @@
x_analytics, webrequest_source, year, month, day, hour])
out.write(result + '\n')
+ if os.path.exists(statFile):
+ os.remove(statFile)
+ os.rename(tmpFile, statFile)
+
+
def run(self):
self.processLogFiles()
diff --git a/scripts/run-hivezero.sh b/scripts/run-hivezero.sh
index ab81156..d1f799b 100755
--- a/scripts/run-hivezero.sh
+++ b/scripts/run-hivezero.sh
@@ -1,14 +1,16 @@
#!/bin/bash
-# ./run-clone.sh 515-05 2014 10 11 0 23
-if [[ -z "$6" ]]; then
- last=$5
-else
+# $1 $2 $3 $4 $5 %6 $7
+# ./run-clone.sh wmf_raw.webrequest 515-05 2014 10 11 0 23
+
+if [[ -z "$7" ]]; then
last=$6
+else
+ last=$7
fi
-for ((hour = $5; hour <= $last; hour++)); do
- printf -v t "tmp_%04d_%02d_%02d_%02d" $2 $3 $4 $hour
- echo hive -f clone-xcs.hql -d "xcs="$1 -d "year="$2 -d "month="$3 -d
"day="$4 -d "hour="$hour -d "table="$t
- export HADOOP_HEAPSIZE=1024 && hive -f clone-xcs.hql -d "xcs="$1 -d
"year="$2 -d "month="$3 -d "day="$4 -d "hour="$hour -d "table="$t
+for ((hour = $6; hour <= $last; hour++)); do
+ printf -v t "tmp_%04d_%02d_%02d_%02d" $3 $4 $5 $hour
+ echo hive -f clone-xcs.hql -d "table="$1 -d "xcs="$2 -d "year="$3 -d
"month="$4 -d "day="$5 -d "hour="$hour -d "table="$t
+ export HADOOP_HEAPSIZE=1024 && hive -f clone-xcs.hql -d "xcs="$2 -d
"year="$3 -d "month="$4 -d "day="$5 -d "hour="$hour -d "table="$t
done
diff --git a/scripts/zero-counts.hql b/scripts/zero-counts.hql
index cccf04d..54d303e 100644
--- a/scripts/zero-counts.hql
+++ b/scripts/zero-counts.hql
@@ -5,9 +5,10 @@
-- Extracts zero stats from webrequests into a separate table
--
-- Usage:
--- hive -f zero-counts.hql -d year=2014 -d month=9 -d day=15 -d
date=2014-09-15
+-- hive -f zero-counts.hql -d table=wmf_raw.webrequest -d year=2014 -d
month=9 -d day=15 -d date=2014-09-15
-- Date is duplicated because I haven't figured an easy way to set
date=printf()
--
+-- set hivevar:table=wmf_raw.webrequest
-- set hivevar:year=2014;
-- set hivevar:month=10;
-- set hivevar:day=21;
@@ -48,7 +49,7 @@
regexp_extract(uri_host,
'^([A-Za-z0-9-]+)(\\.(zero|m))?\\.([a-z]*)\\.org$', 3) subdomain,
regexp_extract(uri_host,
'^([A-Za-z0-9-]+)(\\.(zero|m))?\\.([a-z]*)\\.org$', 4) site
- FROM wmf_raw.webrequest
+ FROM ${table}
WHERE
webrequest_source IN ('text', 'mobile')
AND year=${year}
--
To view, visit https://gerrit.wikimedia.org/r/169750
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib4a75794769c4aaaf155718cb9f1151a1ef4266b
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
