Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/154235
Change subject: Safer cache saving, sms log messages
......................................................................
Safer cache saving, sms log messages
Change-Id: I2f74727154bad255f192e64111fd46bae14e39a8
---
M scripts/logprocessor.py
M scripts/smslogs.py
2 files changed, 16 insertions(+), 13 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/zero-sms
refs/changes/35/154235/1
diff --git a/scripts/logprocessor.py b/scripts/logprocessor.py
index 5a0dc69..bf26f5a 100644
--- a/scripts/logprocessor.py
+++ b/scripts/logprocessor.py
@@ -35,11 +35,15 @@
def writeData(filename, data, columns, separator=u'\t'):
colCount = len(columns)
- with io.open(filename, 'w', encoding='utf8', errors='ignore') as out:
+ tmpFile = filename + '.tmp'
+ with io.open(tmpFile, 'w', encoding='utf8', errors='ignore') as out:
out.writelines(
chain(
[joinValues(columns, separator) + '\n'], # header
imap(lambda vals: joinValues(vals, separator, colCount) +
'\n', data)))
+ if os.path.exists(filename):
+ os.remove(filename)
+ os.rename(tmpFile, filename)
def readData(filename, colCount=0, separator=u'\t'):
@@ -51,8 +55,8 @@
"""
if type(colCount) is list:
colCount = len(colCount)
- isFirst = colCount > 0
- if not isFirst:
+ skipFirst = colCount > 0
+ if not skipFirst:
colCount = -colCount
with io.open(filename, 'r', encoding='utf8', errors='ignore') as inp:
for line in inp:
@@ -60,8 +64,8 @@
if 0 < colCount != len(vals):
raise ValueError('This value should have %d columns, not %d:
%s in file %s' %
(colCount, len(vals), joinValues(vals, u','),
filename))
- if isFirst:
- isFirst = False
+ if skipFirst:
+ skipFirst = False
continue
yield vals
diff --git a/scripts/smslogs.py b/scripts/smslogs.py
index 48885f3..40cadf9 100644
--- a/scripts/smslogs.py
+++ b/scripts/smslogs.py
@@ -124,27 +124,26 @@
if key.size == 0:
skipReason = u'Skipping empty file %s' % filename
elif not fileExists:
- dlReason = u'Downloading new file %s' % filename
+ dlReason = u"it doesn't exist"
elif key.size != os.stat(filePath).st_size:
- dlReason = u'The local size {0:s} <> remote {1:s} for file
{2:s}'.format(
+ dlReason = u'local size %s <> remote %s' % (
locale.format(u"%d", os.stat(filePath).st_size,
grouping=True),
- locale.format(u"%d", key.size, grouping=True),
- filename)
+ locale.format(u"%d", key.size, grouping=True))
elif fileDate and self.downloadIfAfter and fileDate >
self.downloadIfAfter:
- dlReason = u'Re-downloading {0:s} since its date is too close
to last file date {1:s}'\
- .format(filename, self.downloadIfAfter)
+ dlReason = u'date is too close to last file date %s' %
self.downloadIfAfter
else:
skipReason = True
if not self.settings.enableDownloadOld and not fileDate:
- safePrint(u'Skipping legacy-named file %s even though %s' %
(filename, dlReason))
+ if isinstance(dlReason, basestring):
+ safePrint(u'Skipping legacy-named file %s even though %s'
% (filename, dlReason))
continue
if skipReason:
if isinstance(skipReason, basestring):
safePrint(skipReason)
continue
- safePrint(dlReason)
+ safePrint(u'Downloading %s because %s' % (filename, dlReason))
if fileExists:
if os.stat(filePath).st_size == 0:
safePrint(u'Removing empty file %s' % filePath)
--
To view, visit https://gerrit.wikimedia.org/r/154235
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2f74727154bad255f192e64111fd46bae14e39a8
Gerrit-PatchSet: 1
Gerrit-Project: analytics/zero-sms
Gerrit-Branch: master
Gerrit-Owner: Yurik <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
