scipy: http://www.scipy.org/ t-test: http://en.wikipedia.org/wiki/Student's_t-test
scipy and numpy modules are needed. Use unpaired T-test to compare two samples, user can check p-value to know if regression bug exists. If the difference of two samples is considered to be not statistically significant(p <= 0.05), it will add a '+' or '-' before p-value. '+': avg_sample1 < avg_sample2 '-': avg_sample1 > avg_sample2 And split the result to different file. Signed-off-by: Amos Kong <[email protected]> --- client/tools/analyzer.py | 95 ++++++++++++++++++++++++++++++++------------ client/tools/regression.py | 5 +- 2 files changed, 73 insertions(+), 27 deletions(-) diff --git a/client/tools/analyzer.py b/client/tools/analyzer.py index 9fa83f8..c280f5c 100644 --- a/client/tools/analyzer.py +++ b/client/tools/analyzer.py @@ -1,4 +1,4 @@ -import sys, re, string, time, commands, os, random +import sys, re, string, commands, os, random, warnings def tee(content, filename): """ Write content to standard output and file """ @@ -16,16 +16,16 @@ class samples(): fd.close() def getAvg(self): - return self._process(self.files_dict, self._get_list_avg) + return self._process_files(self.files_dict, self._get_list_avg) def getAvgPercent(self, avgs_dict): - return self._process(avgs_dict, self._get_augment_rate) + return self._process_files(avgs_dict, self._get_augment_rate) def getSD(self): - return self._process(self.files_dict, self._get_list_sd) + return self._process_files(self.files_dict, self._get_list_sd) def getSDRate(self, sds_dict): - return self._process(sds_dict, self._get_rate) + return self._process_files(sds_dict, self._get_rate) def _get_rate(self, data): """ num2 / num1 """ @@ -68,36 +68,77 @@ class samples(): sum = 0 for i in data: sum += float(i) - if "." in data[0]: + if "." in str(data[0]): return "%.2f" % (sum / len(data)) return "%d" % (sum / len(data)) - def _process_lines(self, files_dict, row, func): + + def _get_line_list(self, data): + return data + + def getTtestPvalue(self, dict1, dict2): + from scipy import stats + import numpy as np + + s1 = self._process_files(dict1, self._get_line_list, merge=False) + s2 = self._process_files(dict2, self._get_line_list, merge=False) + ret = [] + for line in range(len(s1)): + tmp = [] + if type(s1[line]) != list: + tmp = s1[line] + else: + for raw in range(len(s1[line])): + avg1 = self._get_list_avg(s1[line][raw]) + avg2 = self._get_list_avg(s2[line][raw]) + sample1 = np.array(s1[line][raw]) + sample2 = np.array(s2[line][raw]) + warnings.simplefilter("ignore", RuntimeWarning) + (t, p) = stats.ttest_ind(sample1, sample2) + str = " %.3f" % p + if p <= 0.05: + str = "+%.3f" % p + if avg1 > avg2 : + str = "-%.3f" % p + tmp.append(str) + tmp = "|".join(tmp) + ret.append(tmp) + return ret + + def _process_lines(self, files_dict, row, func, merge): """ Process lines of different sample files with assigned method """ lines = [] - ret_lines = [] + ret = [] for i in range(len(files_dict)): lines.append(files_dict[i][row].split("|")) for col in range(len(lines[0])): data_list = [] for i in range(len(lines)): - data_list.append(lines[i][col].strip()) - ret_lines.append(func(data_list)) - return "|".join(ret_lines) - - def _process(self, files_dict, func): + tmp = lines[i][col].strip() + if "." in tmp: + tmp = float(tmp) + else: + tmp = int(tmp) + data_list.append(tmp) + ret.append(func(data_list)) + if merge: + return "|".join(ret) + return ret + + def _process_files(self, files_dict, func, merge=True): """ Process dicts of sample files with assigned method """ ret_lines = [] for i in range(len(files_dict[0])): if re.findall("[a-zA-Z]", files_dict[0][i]): ret_lines.append(files_dict[0][i].strip()) else: - line = self._process_lines(files_dict, i, func) + line = self._process_lines(files_dict, i, func, merge) ret_lines.append(line) return ret_lines + def display(lists, rate, f, summary="Augment Rate", prefix="% ", ignore_col=1): """ Display lists data to standard format @@ -140,11 +181,9 @@ def display(lists, rate, f, summary="Augment Rate", prefix="% ", ignore_col=1): tee(prefix + rate[i], f) -def analyze(sample_list1, sample_list2, log_file="./result.txt"): +def analyze(sample_list1, sample_list2, log_prefix="./perf-result"): """ Compute averages of two lists of files, compare and display results """ - - commands.getoutput("rm -f %s" % log_file) - tee(time.ctime(time.time()), log_file) + commands.getoutput("rm -f %s-*" % log_prefix) s1 = samples(sample_list1.split()) avg1 = s1.getAvg() sd1 = s1.getSD() @@ -153,18 +192,24 @@ def analyze(sample_list1, sample_list2, log_file="./result.txt"): sd2 = s2.getSD() sd1 = s1.getSDRate([avg1, sd1]) sd2 = s1.getSDRate([avg2, sd2]) - display([avg1], sd1, log_file, summary="Avg1 SD Augment Rate", - prefix="%SD ") - display([avg2], sd2, log_file, summary="Avg2 SD Augment Rate", - prefix="%SD ") + + display([avg1], sd1, log_prefix + "-avgsd.txt", + summary="Avg1 SD Augment Rate", prefix="%SD ") + display([avg2], sd2, log_prefix + "-avgsd.txt", + summary="Avg2 SD Augment Rate", prefix="%SD ") + avgs_rate = s1.getAvgPercent([avg1, avg2]) - display([avg1, avg2], avgs_rate, log_file, summary="AvgS Augment Rate", - prefix="% ") + display([avg1, avg2], avgs_rate, log_prefix + "-avg.txt", + summary="AvgS Augment Rate", prefix="% ") + + tp = s1.getTtestPvalue(s1.files_dict, s2.files_dict) + display([avg1, avg2], tp, log_prefix + "-ttest.txt", + summary="T-test Pvalue", prefix="%PV ") if __name__ == "__main__": if len(sys.argv) < 3: - print 'Usage: python %s "$results list1" "$results list2" $log_file'\ + print 'Usage: python %s "$results list1" "$results list2" $log_prefix'\ % sys.argv[0] sys.exit(1) analyze(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/client/tools/regression.py b/client/tools/regression.py index c550f6f..a467dfe 100644 --- a/client/tools/regression.py +++ b/client/tools/regression.py @@ -1,7 +1,8 @@ import ConfigParser, sys, commands, os import analyzer -def compare(testname, olddir, curdir, config_file='perf.conf', output_dir="./result.txt"): +def compare(testname, olddir, curdir, config_file='perf.conf', + log_prefix="./perf-result"): config = ConfigParser.ConfigParser() config.read(config_file) result_file_pattern = config.get(testname, "result_file_pattern") @@ -14,7 +15,7 @@ def compare(testname, olddir, curdir, config_file='perf.conf', output_dir="./res oldlist = search_files(olddir) newlist = search_files(curdir) if oldlist != "" or newlist != "": - analyzer.analyze(oldlist, newlist, output_dir) + analyzer.analyze(oldlist, newlist, log_prefix) if __name__ == "__main__": _______________________________________________ Autotest mailing list [email protected] http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
