Adamw has submitted this change and it was merged.
Change subject: split out test results functionality
......................................................................
split out test results functionality
Change-Id: Ibacee6c959f07eb65f5ddfc69566ae627c5dedf2
---
M live_analysis/config.py.example
M live_analysis/fr/centralnotice.py
M live_analysis/fr/impressions.py
M live_analysis/fr/tests/__init__.py
M live_analysis/fr/tests/campaign_log.py
A live_analysis/fr/tests/confidence.py
A live_analysis/fr/tests/results.py
M live_analysis/fr/tests/spec.py
8 files changed, 164 insertions(+), 162 deletions(-)
Approvals:
Adamw: Verified; Looks good to me, approved
diff --git a/live_analysis/config.py.example b/live_analysis/config.py.example
index 0bae68f..d4eb178 100644
--- a/live_analysis/config.py.example
+++ b/live_analysis/config.py.example
@@ -37,3 +37,8 @@
results_prefix = ''
reference_day = '20121116000000'
+
+fr_banner_naming =
r'B13_\d+_(?P<testname>[^_]+)_(?P<variation>[^_]+)_(?P<dropdown>[^_]+)_(?P<language>[a-z]{2})(?P<country>[A-Z]{2})'
+
+fudge_trials = 100000
+confidence_level = 0.95
diff --git a/live_analysis/fr/centralnotice.py
b/live_analysis/fr/centralnotice.py
index 4133d29..3e40041 100644
--- a/live_analysis/fr/centralnotice.py
+++ b/live_analysis/fr/centralnotice.py
@@ -25,6 +25,7 @@
} )
if campaign in result:
+ result[campaign]['name'] = campaign
cached_campaigns[campaign] = result[campaign]
return cached_campaigns[campaign]
diff --git a/live_analysis/fr/impressions.py b/live_analysis/fr/impressions.py
index ccc7656..232df64 100644
--- a/live_analysis/fr/impressions.py
+++ b/live_analysis/fr/impressions.py
@@ -1,4 +1,4 @@
-def get_impressions(campaign=None, banner=None):
+def get_impressions(campaign=None, banner=None, **ignore):
import db
query = db.Query()
diff --git a/live_analysis/fr/tests/__init__.py
b/live_analysis/fr/tests/__init__.py
index a4d22eb..b82f36d 100644
--- a/live_analysis/fr/tests/__init__.py
+++ b/live_analysis/fr/tests/__init__.py
@@ -4,54 +4,33 @@
These are not unit tests ;) they are WMF Fundraising A/B tests.
'''
-import re
-
from fr.centralnotice import get_campaign
-from fr.contributions import get_totals
-from fr.impressions import get_impressions
-
-FR_LABEL_PATTERN =
r'B13_\d+_(?P<testname>[^_]+)_(?P<variation>[^_]+)_(?P<dropdown>[^_]+)_(?P<language>[a-z]{2})(?P<country>[A-Z]{2})'
-FUDGE_TRIALS = 100000
-CONFIDENCE_LEVEL = 0.95
+from fr.tests.results import get_banner_results
class FrTest(object):
- def __init__(self, label=None, type="", campaigns=None, banners=None,
start=None, end=None, disabled=False, **ignore):
- #print "Warning: ignoring columns: %s" % (", ".join(ignore.keys()), )
+ def __init__(self, label=None, type="", campaign=None, banners=None,
start=None, end=None, disabled=False, **ignore):
+ print "Warning: ignoring columns: %s" % (", ".join(ignore.keys()), )
- self.campaigns = []
- if campaigns:
- if hasattr(campaigns, 'split'):
- campaigns = [ s.strip() for s in campaigns.split(",") ]
- for name in campaigns:
- campaign = get_campaign(name)
- if campaign:
- campaign['name'] = name
- self.campaigns.append(campaign)
- else:
- print "Warning: no such campaign '%s'" % name
+ self.campaign = get_campaign(campaign)
+ if not self.campaign:
+ print "Warning: no such campaign '%s'" % campaign
- type = type.lower()
+ self.type = type.lower()
- if type.count('banner') > 0:
+ if self.type.count('banner') > 0:
self.is_banner_test = True
if banners:
if hasattr(banners, 'strip'):
banners = [ s.strip() for s in banners.split(",") ]
self.banners = banners
else:
- def reduce_banners(sum, campaign):
- if campaign['banners']:
- sum.extend(campaign['banners'].keys())
- return sum
+ if self.campaign['banners']:
+ self.banners = self.campaign['banners'].keys()
- self.banners = reduce(reduce_banners, self.campaigns, [])
+ #self.variations = [ FrTestVariation(banner=name) for name in
self.banners ]
- #self.variations = [ FrTestVariation(banner=b) for b in
self.banners ]
-
- self.is_country_test = (type.count('country') > 0)
- self.is_lp_test = (type.count('lp') > 0)
-
- self.type = type
+ self.is_country_test = (self.type.count('country') > 0)
+ self.is_lp_test = (self.type.count('lp') > 0)
self.start_time = start
self.end_time = end
@@ -63,61 +42,24 @@
self.results = []
def load_results(self):
- for campaign in self.campaigns:
- if self.is_banner_test and self.banners:
- results = []
- for name in self.banners:
- test_case = self.get_case(
- campaign=campaign['name'],
- banner=name
- )
- totals = get_totals(**test_case)
- impressions = get_impressions(campaign=campaign['name'],
banner=name)
+ if self.is_banner_test and self.banners:
+ cases = []
+ for name in self.banners:
+ test_case = self.get_case(
+ campaign=self.campaign['name'],
+ banner=name
+ )
+ cases.append(test_case)
- result_extra = {
- 'preview':
"http://en.wikipedia.org/wiki/Special:Random?banner="; + name,
- 'screenshot':
"http://fundraising-archive.wmflabs.org/banner/%s.png"; % name,
+ self.results.extend(get_banner_results(cases))
- 'impressions': str(impressions),
- }
-
- # FIXME: refactor to a variations hook
- match = re.match(FR_LABEL_PATTERN, name)
- if match:
- result_extra.update({
- 'label': match.group("testname"),
- 'language': match.group("language"),
- 'variation': match.group("variation"),
- 'dropdown': match.group("dropdown") is "dr",
- 'country': match.group("country"),
-
- 'preview':
"http://en.wikipedia.org/wiki/Special:Random?banner=%s&country=%s&uselang=%s"; %
(name, match.group("country"), match.group("language")),
- })
-
- results.append(TestResult(
- criteria=test_case,
- results=[totals, result_extra]
- ))
-
- try:
- confidence = self.get_confidence(results, 'banner',
'donations')
- if confidence:
- for i, levels in enumerate(confidence):
- results[i].add_result('p-value',
levels.two_tailed_p_value)
- results[i].add_result('improvement',
levels.relative_improvement.value * 100)
- except ImportError as e:
- print "ERROR: not calculating confidence, dummy: ",
e.message
- results[0].add_result('confidencelink',
self.get_confidence_link(results, 'banner', 'donations', FUDGE_TRIALS))
-
- self.results.extend(results)
-
- if self.is_country_test:
- results = [ calculate_result(country=code) for code in
campaign['countries'] ]
- self.results.extend(results)
+ if self.is_country_test:
+ #results = [ calculate_result(country=code) for code in
campaign['countries'] ]
+ #self.results.extend(results)
+ print "country test type not implemented"
if self.is_lp_test:
print "LP test type not implemented"
- pass
def get_case(self, **kw):
conditions = {
@@ -131,7 +73,12 @@
def __repr__(self):
description = '''
Test: %(label)s (%(campaigns)s) %(start)s - %(end)s
-''' % {'label': self.label, 'campaigns': str([c['name'] for c in
self.campaigns]), 'start': self.start_time, 'end': self.end_time, }
+''' % {
+ 'label': self.label,
+ 'campaigns': self.campaign['name'],
+ 'start': self.start_time,
+ 'end': self.end_time,
+ }
if not self.enabled:
description += " DISABLED "
if self.is_banner_test:
@@ -142,76 +89,4 @@
description += " lps: " + str(self.lps)
return description
- def get_confidence(self, results, name_column=None, successes_column=None,
trials=None):
- from stats_abba import Experiment
- num_test_cases = len(results)
-
- if not num_test_cases:
- return
-
- results = sorted(results, key=lambda result:
result.results[successes_column])
- for result in results:
- if result.results[successes_column]:
- baseline_successes = result.results[successes_column]
- break
-
- if not baseline_successes:
- return
-
- experiment = Experiment(
- num_trials=FUDGE_TRIALS,
- baseline_num_successes=baseline_successes,
- baseline_num_trials=FUDGE_TRIALS,
- confidence_level=CONFIDENCE_LEVEL
- )
- #useMultipleTestCorrection=true
-
- cases = []
- for result in results:
- name = result.results[name_column]
- successes = result.results[successes_column]
- if hasattr(trials, 'encode'):
- trials = result.results[trials]
- else:
- trials = FUDGE_TRIALS
- calculated = experiment.get_results(num_successes=successes,
num_trials=trials)
- cases.append(calculated)
-
- return cases
-
- def get_confidence_link(self, results, name_column, successes_column,
trials):
- cases = []
- for result in results:
- # skip empty results, usually these will be "blank" banners
- if not result.results[successes_column]:
- continue
- name = result.results[name_column]
- successes = result.results[successes_column]
- if hasattr(trials, 'encode'):
- trials = result.results[trials]
- cases.append( "%s=%s,%s" % (name, successes, trials) )
- return
"http://www.thumbtack.com/labs/abba/#%s&abba:intervalConfidenceLevel=0.95&abba:useMultipleTestCorrection=true";
% "&".join(cases)
-
-class TestResult(object):
- def __init__(self, criteria=None, results=None):
- self.criteria = criteria
- self.results = {}
- if results:
- self.add_result(results)
-
- def add_result(self, result, value=None):
- if hasattr(result, 'keys'):
- self.results.update(result)
- elif hasattr(result, 'append'):
- for entry in result:
- self.add_result(entry)
- else:
- self.results[result] = value
-
- def __repr__(self):
- import json
- return '''
-Result: %s
- -> %s''' % (json.dumps(self.criteria, indent=4), json.dumps(self.results,
indent=4), )
-
-#class TestVariation(object):
+#class FrTestVariation(object):
diff --git a/live_analysis/fr/tests/campaign_log.py
b/live_analysis/fr/tests/campaign_log.py
index cc15330..2b0dc50 100644
--- a/live_analysis/fr/tests/campaign_log.py
+++ b/live_analysis/fr/tests/campaign_log.py
@@ -31,7 +31,7 @@
return FrTest(
type="banner",
- campaigns=entry['campaign'],
+ campaign=entry['campaign'],
banners=banners,
start=start,
end=end
diff --git a/live_analysis/fr/tests/confidence.py
b/live_analysis/fr/tests/confidence.py
new file mode 100644
index 0000000..2c02f31
--- /dev/null
+++ b/live_analysis/fr/tests/confidence.py
@@ -0,0 +1,67 @@
+import config
+from stats_abba import Experiment
+
+def add_confidence(results, name_column, successes_column):
+ confidence = get_confidence(results, name_column, successes_column)
+ if confidence:
+ for i, levels in enumerate(confidence):
+ results[i].results.update({
+ 'p-value': levels.two_tailed_p_value,
+ 'improvement': levels.relative_improvement.value * 100,
+ })
+ results[0].results.update({
+ 'confidencelink': get_confidence_link(results, name_column,
successes_column)
+ })
+
+def get_confidence(results, name_column=None, successes_column=None,
trials=None):
+ num_test_cases = len(results)
+
+ if not num_test_cases:
+ return
+
+ results = sorted(results, key=lambda result:
result.results[successes_column])
+ baseline_successes = None
+ for result in results:
+ if result.results[successes_column]:
+ baseline_successes = result.results[successes_column]
+ break
+
+ if not baseline_successes:
+ return
+
+ experiment = Experiment(
+ num_trials=config.fudge_trials,
+ baseline_num_successes=baseline_successes,
+ baseline_num_trials=config.fudge_trials,
+ confidence_level=config.confidence_level
+ )
+ #useMultipleTestCorrection=true
+
+ cases = []
+ for result in results:
+ name = result.results[name_column]
+ successes = result.results[successes_column]
+ if hasattr(trials, 'encode'):
+ trials = result.results[trials]
+ else:
+ trials = config.fudge_trials
+ calculated = experiment.get_results(num_successes=successes,
num_trials=trials)
+ cases.append(calculated)
+
+ return cases
+
+def get_confidence_link(results, name_column, successes_column):
+ cases = []
+ for result in results:
+ # skip empty results, usually these will be "blank" banners
+ if not result.results[successes_column]:
+ continue
+ name = result.results[name_column]
+ successes = result.results[successes_column]
+ cases.append( "%s=%s,%s" % (name, successes, config.fudge_trials) )
+
+ return
"http://www.thumbtack.com/labs/abba/#%(cases)s&abba:intervalConfidenceLevel=%(confidence)s&abba:useMultipleTestCorrection=true"
% {
+ 'cases': "&".join(cases),
+ 'confidence': config.confidence_level,
+ }
+
diff --git a/live_analysis/fr/tests/results.py
b/live_analysis/fr/tests/results.py
new file mode 100644
index 0000000..8179732
--- /dev/null
+++ b/live_analysis/fr/tests/results.py
@@ -0,0 +1,54 @@
+import re
+import json
+
+import config
+from fr.contributions import get_totals
+from fr.impressions import get_impressions
+from fr.tests.confidence import add_confidence
+
+class TestResult(object):
+ def __init__(self, criteria=None, results={}):
+ self.criteria = criteria
+ self.results = results
+
+ def __repr__(self):
+ return '''
+Result: %s
+ -> %s''' % (
+ json.dumps(self.criteria, indent=4),
+ json.dumps(self.results, indent=4),
+ )
+
+
+def get_banner_results(cases):
+ results = [ banner_results(case) for case in cases ]
+
+ add_confidence(results, 'banner', 'donations')
+
+ return results
+
+def banner_results(criteria):
+ results = get_totals(**criteria)
+ impressions = get_impressions(**criteria)
+
+ results.update({
+ 'preview':
"http://en.wikipedia.org/wiki/Special:Random?banner=%s&reset=1"; %
criteria['banner'],
+ 'screenshot': "http://fundraising-archive.wmflabs.org/banner/%s.png"; %
criteria['banner'],
+
+ 'impressions': str(impressions),
+ })
+
+ # FIXME: refactor to a variations hook
+ match = re.match(config.fr_banner_naming, criteria['banner'])
+ if match:
+ results.update({
+ 'label': match.group("testname"),
+ 'language': match.group("language"),
+ 'variation': match.group("variation"),
+ 'dropdown': match.group("dropdown") is "dr",
+ 'country': match.group("country"),
+
+ 'preview':
"http://en.wikipedia.org/wiki/Special:Random?banner=%s&country=%s&uselang=%s&reset=1";
% (criteria['banner'], match.group("country"), match.group("language")),
+ })
+
+ return TestResult(criteria, results)
diff --git a/live_analysis/fr/tests/spec.py b/live_analysis/fr/tests/spec.py
index dd8e5b0..e121617 100644
--- a/live_analysis/fr/tests/spec.py
+++ b/live_analysis/fr/tests/spec.py
@@ -13,11 +13,11 @@
yield FrTest(**row)
def compare_test_fuzzy(a, b):
- if a.campaigns == b.campaigns and a.banners == b.banners:
+ if a.campaign == b.campaign and a.banners == b.banners:
return True
def is_fr_test(test):
- if test.label and test.banners and test.campaigns:
+ if test.label and test.banners and test.campaign:
is_chapter = re.search(r'(_|\b)WM[A-Z]{2}(_|\b)', test.banners[0])
return not is_chapter
--
To view, visit https://gerrit.wikimedia.org/r/48844
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ibacee6c959f07eb65f5ddfc69566ae627c5dedf2
Gerrit-PatchSet: 3
Gerrit-Project: wikimedia/fundraising/tools
Gerrit-Branch: master
Gerrit-Owner: Adamw <[email protected]>
Gerrit-Reviewer: Adamw <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
![http://fundraising-archive.wmflabs.org/banner/%s.png"](http://fundraising-archive.wmflabs.org/banner/%s.png"){kind=link}