From: Ian Romanick <ian.d.roman...@intel.com>
Also perform a T-test on the helped and hurt, and provide a
recommendation based on the result of the T-test. I have seen several
instances where the result of the T-test is the opposite of what you
would expect by looking at the other logged data, so I'm not sure how
useful / correct this is.
Signed-off-by: Ian Romanick <ian.d.roman...@intel.com>
---
report.py | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 84 insertions(+), 1 deletion(-)
diff --git a/report.py b/report.py
index 175a953..d10cb84 100755
--- a/report.py
+++ b/report.py
@@ -2,6 +2,9 @@
import re
import argparse
+import statistics
+from scipy import stats
+import numpy
def get_results(filename):
@@ -63,6 +66,27 @@ def get_result_string(p, b, a):
def split_list(string):
return string.split(",")
+
+def gather_statistics(changes, before, after, m):
+ stats = (0.0, 0, 0.0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0)
+
+ num = len(changes)
+ if num > 0:
+ absolute = [abs(before[p][m] - after[p][m]) for p in changes]
+ relative = [0 if before[p][m] == 0 else abs(before[p][m] -
after[p][m]) / before[p][m] for p in changes]
+
+ stats = (statistics.mean(absolute),
+ statistics.median(absolute),
+ min(absolute),
+ max(absolute),
+ statistics.mean(relative),
+ statistics.median(relative),
+ min(relative),
+ max(relative))
+
+ return stats
+
+
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--measurements", "-m", type=split_list,
@@ -82,6 +106,9 @@ def main():
affected_after = {}
num_hurt = {}
num_helped = {}
+ helped_statistics = {}
+ hurt_statistics = {}
+ t_test = {}
for m in args.measurements:
total_before[m] = 0
@@ -135,9 +162,18 @@ def main():
if len(hurt) > 0:
print("")
+ helped_statistics[m] = gather_statistics(helped, args.before,
args.after, m)
+ hurt_statistics[m] = gather_statistics(hurt, args.before, args.after,
m)
+
num_helped[m] = len(helped)
num_hurt[m] = len(hurt)
+ if num_hurt[m] >= 2 and num_helped[m] >= 2:
+ A = [abs(args.before[p][m] - args.after[p][m]) for p in helped]
+ B = [abs(args.before[p][m] - args.after[p][m]) for p in hurt]
+ C = [0 if args.before[p][m] == 0 else abs(args.before[p][m] -
args.after[p][m]) / args.before[p][m] for p in helped]
+ D = [0 if args.before[p][m] == 0 else abs(args.before[p][m] -
args.after[p][m]) / args.before[p][m] for p in hurt]
+ t_test[m] = stats.ttest_ind(A, B) + stats.ttest_ind(C, D)
lost = []
gained = []
@@ -172,13 +208,60 @@ def main():
print("total {0} in shared programs: {1}\n"
"{0} in affected programs: {2}\n"
"helped: {3}\n"
- "HURT: {4}\n".format(
+ "HURT: {4}".format(
m,
change(total_before[m], total_after[m]),
change(affected_before[m], affected_after[m]),
num_helped[m],
num_hurt[m]))
+ # Statistics for spills and fills is usually meaningless.
+ if m in ["spills", "fills"]:
+ print()
+ continue
+
+ if num_helped[m] > 2 or (num_helped[m] > 0 and num_hurt[m] > 0):
+ (avg_abs, med_abs, lo_abs, hi_abs, avg_rel, med_rel, lo_rel,
hi_rel) = helped_statistics[m]
+
+ print("helped stats (abs) min: {} max: {} x\u0304: {:.2f}
x\u0303: {}".format(
+ lo_abs, hi_abs, avg_abs, int(med_abs)))
+ print("helped stats (rel) min: {} max: {} x\u0304: {} x\u0303:
{}".format(
+ format_percent(lo_rel),
+ format_percent(hi_rel),
+ format_percent(avg_rel),
+ format_percent(med_rel)))
+
+ if num_hurt[m] > 2 or (num_hurt[m] > 0 and num_helped[m] > 0):
+ (avg_abs, med_abs, lo_abs, hi_abs, avg_rel, med_rel, lo_rel,
hi_rel) = hurt_statistics[m]
+
+ print("HURT stats (abs) min: {} max: {} x\u0304: {:.2f}
x\u0303: {}".format(
+ lo_abs, hi_abs, avg_abs, int(med_abs)))
+ print("HURT stats (rel) min: {} max: {} x\u0304: {} x\u0303:
{}".format(
+ format_percent(lo_rel),
+ format_percent(hi_rel),
+ format_percent(avg_rel),
+ format_percent(med_rel)))
+
+ if m in t_test:
+ print("abs t: {}, p: {}".format(t_test[m][0], t_test[m][1]))
+ print("rel t: {}, p: {}".format(t_test[m][2], t_test[m][3]))
+
+ # Be very, very conservative about applying results of the
+ # T-test. Only report a recommendation if the T-tests for
+ # both the absolute data and the relative data agree.
+ if t_test[m][1] < 0.05 and t_test[m][3] < 0.05:
+ abs_diff = helped_statistics[m][0] - hurt_statistics[m][0]
+ rel_diff = helped_statistics[m][4] - hurt_statistics[m][4]
+
+ if abs_diff > 0 and rel_diff > 0:
+ print("Positive result.")
+ elif abs_diff < 0 and rel_diff < 0:
+ print("Negative result.")
+ else:
+ print("Inconclusive result.");
+
+ print()
+
if len(lost) > 0 or len(gained) > 0 or not args.changes_only:
print("LOST: " + str(len(lost)))
print("GAINED: " + str(len(gained)))
--
2.9.5
_______________________________________________
mesa-dev mailing list
mesa-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/mesa-dev
