- Revision
- 11230
- Author
- dan
- Date
- 2006-07-21 14:57:53 -0700 (Fri, 21 Jul 2006)
Log Message
New Test Framwork - Add parameters for passing DEBUG and MASK to logger. Convert to using a easier to read report when DEBUG=0, and add a summary of tests pass/ fail status at the end.
Modified Paths
Diff
Modified: trunk/chandler/application/Utility.py (11229 => 11230)
--- trunk/chandler/application/Utility.py 2006-07-21 20:31:56 UTC (rev 11229) +++ trunk/chandler/application/Utility.py 2006-07-21 21:57:53 UTC (rev 11230) @@ -172,6 +172,8 @@ 'testScripts':('-t', '--testScripts','b', False, None, 'run all test scripts'), 'scriptFile': ('-f', '--scriptFile', 's', None, None, 'script file to execute after startup'), 'chandlerTests': ('', '--chandlerTests', 's', None, None, 'file:TestClass,file2:TestClass2 to be executed by new framework'), + 'chandlerTestDebug': ('-D', '--chandlerTestDebug', 's', 0, None, '0=print only failures, 1=print pass and fail, 2=print pass & fail & check repository after each test'), + 'chandlerTestMask': ('-M', '--chandlerTestMask', 's', 3, None, '0=print all, 1=hide reports, 2=also hide actions, 3=also hide test names'), 'chandlerPerformanceTests': ('', '--chandlerPerformanceTests', 's', None, None, 'file:TestClass,file2:TestClass2 to be executed by performance new framework'), 'chandlerTestLogfile': ('', '--chandlerTestLogfile', 's', None, None, 'file for chandlerTests output'), 'scriptTimeout': ('-s', '--scriptTimeout', 's', 0, None, 'script file timeout'),
Modified: trunk/chandler/parcels/osaf/framework/scripting/script.py (11229 => 11230)
--- trunk/chandler/parcels/osaf/framework/scripting/script.py 2006-07-21 20:31:56 UTC (rev 11229) +++ trunk/chandler/parcels/osaf/framework/scripting/script.py 2006-07-21 21:57:53 UTC (rev 11230) @@ -255,10 +255,13 @@ # Execute new framework if chandlerTests option is called chandlerTests = Globals.options.chandlerTests - logFileName = Globals.options.chandlerTestLogfile if chandlerTests: + logFileName = Globals.options.chandlerTestLogfile + testDebug = Globals.options.chandlerTestDebug + testMask = Globals.options.chandlerTestMask from tools.cats.framework.runTests import run_tests - run_tests(chandlerTests, logName=logFileName) + print 'from script.py debug=%s, mask=%s' % (testDebug, testMask) + run_tests(chandlerTests, debug=testDebug, mask=testMask, logName=logFileName) # Execute new framework if chandlerPerformanceTests option is called chandlerPerformanceTests = Globals.options.chandlerPerformanceTests
Modified: trunk/chandler/tools/cats/framework/TestOutput.py (11229 => 11230)
--- trunk/chandler/tools/cats/framework/TestOutput.py 2006-07-21 20:31:56 UTC (rev 11229) +++ trunk/chandler/tools/cats/framework/TestOutput.py 2006-07-21 21:57:53 UTC (rev 11230) @@ -14,7 +14,7 @@ """TestOutput class for cats 0.2 -This is a module contains on major class, TestOutput, +This is a module contains a major class, TestOutput, which is used for string output, timeing, and report->action->test->suite encapsulation. """ @@ -37,7 +37,7 @@ """ Test output and timing class. """ - def __init__(self, logName=None, debug=0, mask=2, stdout=None): + def __init__(self, logName=None, debug=1, mask=2, stdout=None): """Instantiation method Keyword Arguments: @@ -53,8 +53,9 @@ 3 = don't show report, action, test stdout bool -- Switch to turn on stdout output """ - self.debug = debug - self.mask = mask + self.debug = int(debug) + self.mask = int(mask) + self.logName = logName self.suiteList = [] self.testList = [] self.actionList = [] @@ -73,21 +74,21 @@ self.stdout = sys.__stdout__ else: self.stdout = None - - if logName is not None: - logName = os.path.abspath(logName) + + if self.logName is not None: + self.LogName = os.path.abspath(self.logName) # Rename the last log file to timestamped version try: # getatime returns last mod on Unix, and creation time on Win - atime = os.path.getatime(logName) + atime = os.path.getatime(self.LogName) time_stamp = time.strftime('%Y%m%d%H%M%S', time.localtime(atime)) - os.rename(logName, '%s.%s' % (logName, time_stamp)) + os.rename(self.LogName, '%s.%s' % (self.LogName, time_stamp)) except OSError: # Most likely the file didn't exist, just ignore pass - self.f = file(logName, 'w') + self.f = file(self.logName, 'w') else: self.f = None @@ -300,7 +301,31 @@ return #don't print passes when debug = 0 if level >= self.mask: sendTo_write(string, result) + + def easyReadSummary(self): + """Failures displayed to be easily readable by humans """ + def stripStuff(s): + if s == None: + return '' + if 'Traceback' not in s: + s = s.strip('None').strip('\n') + return s + self._parse_results() + for suite_dict in self.suiteList: + if suite_dict['result'] == False: + self._write('\nSUITE %s FAILED\n' % suite_dict['name']) + for test_dict in suite_dict['testlist']: + if test_dict['result'] == False: + self._write('\tTEST %s FAILED\n ' % stripStuff(test_dict['name'])) + for action_dict in test_dict['actionlist']: + if action_dict['result'] == False: + self._write('\t\tACTION %s %s\n' % (stripStuff(action_dict['name']),stripStuff(action_dict['comment']))) + for report_tuple in action_dict['reportlist']: + if report_tuple[0] is False: + self._write('\t\t\tREPORT %s %s\n' % (stripStuff(report_tuple[1]), stripStuff(report_tuple[2]))) + + def traceback(self): """Method to handle python traceback exception.""" import sys, traceback @@ -314,6 +339,9 @@ if self.stdout is None and self.f is None: traceback.print_exception(type, value, stack, None, self.stderr) + #report the action + self.report(False, 'Traceback detected') + #Clean up logger state if self.inAction is True: self.endAction(result=False, comment='Action Failure due to traceback') @@ -347,22 +375,22 @@ self._parse_results() #Parse through finalized suiteList for failures - self._write('Test Report;\n') + self._write('\n*** Test Report ***\n\n') for suite_dict in self.suiteList: if suite_dict['result'] is False: - self._write('*Suite ""%s"" Failed :: Total Time ""%s"" :: Comment ""%s""\n' % (suite_dict['name'], suite_dict['totaltime'], suite_dict['comment'])) + self._write('Suite ""%s"" Failed :: Total Time ""%s"" :: Comment ""%s""\n' % (suite_dict['name'], suite_dict['totaltime'], suite_dict['comment'])) suites_failed = suites_failed + 1 for test_dict in suite_dict['testlist']: if test_dict['result'] is False: - self._write('**Test ""%s"" Failed :: Total Time ""%s"" :: Comment ""%s""\n' % (test_dict['name'], test_dict['totaltime'], test_dict['comment'])) + self._write(' Test ""%s"" Failed :: Total Time ""%s"" :: Comment ""%s""\n' % (test_dict['name'], test_dict['totaltime'], test_dict['comment'])) tests_failed = tests_failed + 1 for action_dict in test_dict['actionlist']: if action_dict['result'] is False: - self._write('***Action ""%s"" Failed :: Total Time ""%s"" :: Comment ""%s""\n' % (action_dict['name'], action_dict['totaltime'], action_dict['comment'])) + self._write(' Action ""%s"" Failed :: Total Time ""%s"" :: Comment ""%s""\n' % (action_dict['name'], action_dict['totaltime'], action_dict['comment'])) actions_failed = actions_failed + 1 for report_tuple in action_dict['reportlist']: if report_tuple[0] is False: - self._write('****Report ""%s"" Failed :: Comment ""%s""\n' % (report_tuple[1], report_tuple[2])) + self._write(' Report ""%s"" Failed :: Comment ""%s""\n' % (report_tuple[1], report_tuple[2])) reports_failed = reports_failed + 1 #Calculate number ran @@ -378,7 +406,19 @@ self._write('$Suites run=%s, pass=%s, fail=%s :: Tests run=%s, pass=%s, fail=%s :: Actions run=%s, pass=%s, fail=%s :: Reports run=%s, pass=%s, fail=%s \n' % (suites_ran, suites_ran - suites_failed, suites_failed, tests_ran, tests_ran - tests_failed, tests_failed, actions_ran, actions_ran - actions_failed, actions_failed, reports_ran, reports_ran - reports_failed, reports_failed)) - + + def simpleSummary(self): + #write simple summary + self._write("\n*********** TEST RESULTS ************\n*************************************\n") + for suite_dict in self.suiteList: + for test_dict in suite_dict['testlist']: + if test_dict['result'] == False: + self._write('%s*FAILED*\n' % test_dict['name'].ljust(30,'_')) + else: + self._write('%s passed \n' % test_dict['name'].ljust(30,'_')) + self._write("*************************************\n") + + def tinderOutput(self): #write out stuff for tinderbox for suite_dict in self.suiteList: self._write('#TINDERBOX# Testname = %s\n' % suite_dict['name']) @@ -438,8 +478,8 @@ #test masking levels = ['report', 'action', 'test', 'suite'] - for debug in [0, 1, 2]: - for mask in [0, 1, 2, 3]: + for self.debug in [0, 1, 2]: + for self.mask in [0, 1, 2, 3]: for result in [True, False]: print '########## debug = %d :: mask = %d :: result = %s' % (debug, mask, result) logger = TestOutput(debug=debug, mask=mask, stdout=True)
Modified: trunk/chandler/tools/cats/framework/runTests.py (11229 => 11230)
--- trunk/chandler/tools/cats/framework/runTests.py 2006-07-21 20:31:56 UTC (rev 11229) +++ trunk/chandler/tools/cats/framework/runTests.py 2006-07-21 21:57:53 UTC (rev 11230) @@ -32,16 +32,16 @@ logger.addComment('Checking for repository corruption') QAUITestAppLib.App_ns.itsView.check() -def run_tests(tests, logName=None): +def run_tests(tests, debug=0, mask=3, logName=None): """Method to execute cats tests, must be in Functional directory.""" - logger = TestOutput(stdout=True, debug=0, logName=logName) + logger = TestOutput(stdout=True, debug=debug, mask=mask, logName=logName) logger.startSuite(name='ChandlerTestSuite') for paramSet in tests.split(','): try: filenameAndTest = paramSet.split(':') - #dan added this as a convenience, I'm already tired of typing this stuff twice + #assume file name and and test name are the same if only one given if len(filenameAndTest) < 2: filenameAndTest.append(filenameAndTest[0]) teststring = 'from tools.cats.Functional.%s import %s' % (filenameAndTest[0], filenameAndTest[1]) @@ -55,7 +55,12 @@ if logger.debug < 2: checkRepo(logger) logger.endSuite() - logger.summary() + if logger.debug == 0: + logger.easyReadSummary() + else: + logger.summary() + logger.simpleSummary() + logger.tinderOutput() import osaf.framework.scripting as scripting scripting.app_ns().root.Quit() @@ -68,7 +73,7 @@ try: filenameAndTest = paramSet.split(':') - #dan added this as a convenience, I'm already tired of typing this stuff twice + #assume file name and and test name are the same if only one given if len(filenameAndTest) < 2: filenameAndTest.append(filenameAndTest[0]) teststring = 'from tools.cats.Performance.%s import %s' % (filenameAndTest[0], filenameAndTest[1])
_______________________________________________ Commits mailing list [email protected] http://lists.osafoundation.org/mailman/listinfo/commits
