Sebastien's threaded tests for marvin While this sends out workers for each test fixture there seem to be problems with urllib2 and threading. urllib2 being unsafe the threads behaviour is not deterministic. Tests timeout early or are unable to contact the CloudStack end point. Retaining the patch to test with a more thread safe library
Project: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/commit/67bd8ae8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/tree/67bd8ae8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/diff/67bd8ae8 Branch: refs/heads/marvin-parallel Commit: 67bd8ae85480bee82056567e8de2260c6186135e Parents: c86c5d2 Author: Sebastien Goasguen <[email protected]> Authored: Fri Oct 12 18:28:45 2012 +0530 Committer: Prasanna Santhanam <[email protected]> Committed: Fri Oct 12 18:28:45 2012 +0530 ---------------------------------------------------------------------- .../marvin/marvin/ThreadedTestCaseExecuteEngine.py | 123 +++++++++++++++ tools/marvin/marvin/deployAndRun.py | 67 +++++--- 2 files changed, 166 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/67bd8ae8/tools/marvin/marvin/ThreadedTestCaseExecuteEngine.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/ThreadedTestCaseExecuteEngine.py b/tools/marvin/marvin/ThreadedTestCaseExecuteEngine.py new file mode 100644 index 0000000..0230b94 --- /dev/null +++ b/tools/marvin/marvin/ThreadedTestCaseExecuteEngine.py @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest +import xmlrunner +import os +import sys +import logging +from functools import partial +import threading +import Queue + +def testCaseLogger(message, logger=None): + if logger is not None: + logger.debug(message) + +class ThreadedTests(threading.Thread): + + def __init__(self, testqueue, format, testResultLogFile, xmlDir): + threading.Thread.__init__(self) + self.testqueue = testqueue + self.format = format + self.testResultLogFile = testResultLogFile + self.xmlDir = xmlDir + + def run(self): + test=self.testqueue.get() + if self.format == "text": + unittest.TextTestRunner(stream=self.testResultLogFile, verbosity=2).run(test) + elif self.format == "xml": + xmlrunner.XMLTestRunner(output=self.xmlDir, verbose=True).run(test) + + self.testqueue.task_done() + +class TestCaseExecuteEngine(object): + def __init__(self, testclient, testcaseLogFile=None, testResultLogFile=None, format="text", xmlDir="xml-reports",num_threads=1): + """ + Initialize the testcase execution engine, just the basics here + @var testcaseLogFile: client log file + @var testResultLogFile: summary report file + """ + self.testclient = testclient + self.logformat = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s") + self.loader = unittest.loader.TestLoader() + self.suite = None + self.format = format + self.numthreads=num_threads + + if testcaseLogFile is not None: + self.logfile = testcaseLogFile + self.logger = logging.getLogger("TestCaseExecuteEngine") + fh = logging.FileHandler(self.logfile) + fh.setFormatter(self.logformat) + self.logger.addHandler(fh) + self.logger.setLevel(logging.DEBUG) + if testResultLogFile is not None: + ch = logging.StreamHandler() + ch.setLevel(logging.ERROR) + ch.setFormatter(self.logformat) + self.logger.addHandler(ch) + fp = open(testResultLogFile, "w") + self.testResultLogFile = fp + else: + self.testResultLogFile = sys.stdout + if self.format == "xml" and (xmlDir is not None): + self.xmlDir = xmlDir + + def loadTestsFromDir(self, testDirectory): + """ Load the test suites from a package with multiple test files """ + self.suite = self.loader.discover(testDirectory) + self.injectTestCase(self.suite) + + def loadTestsFromFile(self, file_name): + """ Load the tests from a single script/module """ + if os.path.isfile(file_name): + self.suite = self.loader.discover(os.path.dirname(file_name), os.path.basename(file_name)) + self.injectTestCase(self.suite) + + def injectTestCase(self, testSuites): + for test in testSuites: + if isinstance(test, unittest.BaseTestSuite): + self.injectTestCase(test) + else: + #logger bears the name of the test class + testcaselogger = logging.getLogger("testclient.testcase.%s"%test.__class__.__name__) + fh = logging.FileHandler(self.logfile) + fh.setFormatter(self.logformat) + testcaselogger.addHandler(fh) + testcaselogger.setLevel(logging.DEBUG) + + #inject testclient and logger into each unittest + setattr(test, "testClient", self.testclient) + setattr(test, "debug", partial(testCaseLogger, logger=testcaselogger)) + setattr(test.__class__, "clstestclient", self.testclient) + if hasattr(test, "UserName"): + self.testclient.createNewApiClient(test.UserName, test.DomainName, test.AcctType) + + def run(self): + testqueue=Queue.Queue() + for test in self.suite: + testqueue.put(test) + + thread_list=[] + for i in range(num_threads): + thread_list.append(ThreadedTests(testqueue, self.format, self.testResultLogFile, self.xmlDir)) + + for thread in thread_list: + thread.start() + testqueue.join() http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/67bd8ae8/tools/marvin/marvin/deployAndRun.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/deployAndRun.py b/tools/marvin/marvin/deployAndRun.py index 1c82d9f..21639d7 100644 --- a/tools/marvin/marvin/deployAndRun.py +++ b/tools/marvin/marvin/deployAndRun.py @@ -17,50 +17,69 @@ import deployDataCenter import TestCaseExecuteEngine -from optparse import OptionParser +import ThreadedTestCaseExecuteEngine +from argparse import ArgumentParser import os if __name__ == "__main__": - parser = OptionParser() #TODO: deprecate and use the argparse module + parser = ArgumentParser() - parser.add_option("-c", "--config", action="store", default="./datacenterCfg", dest="config", help="the path where the json config file generated, by default is ./datacenterCfg") - parser.add_option("-d", "--directory", dest="testCaseFolder", help="the test case directory") - parser.add_option("-r", "--result", dest="result", help="test result log file") - parser.add_option("-t", "--client", dest="testcaselog", help="test case log file") - parser.add_option("-l", "--load", dest="load", action="store_true", help="only load config, do not deploy, it will only run testcase") - parser.add_option("-f", "--file", dest="module", help="run tests in the given file") - parser.add_option("-x", "--xml", dest="xmlrunner", help="use the xml runner to generate xml reports and path to store xml files") - (options, args) = parser.parse_args() + parser.add_argument("-c", "--config", action="store", default="./datacenterCfg", dest="config", + help="The path where the json config file generated, by default is ./datacenterCfg") + parser.add_argument("-d", "--directory", dest="testCaseFolder", + help="The test case directory") + parser.add_argument("-r", "--result", dest="result", + help="Test result log file") + parser.add_argument("-t", "--client", dest="testcaselog", + help="Test case log file") + parser.add_argument("-l", "--load", dest="load", action="store_true", + help="Only load config, do not deploy, it will only run testcase") + parser.add_argument("-f", "--file", dest="module", + help="Run tests in the given file") + parser.add_argument("-x", "--xml", dest="xmlrunner", + help="Use the xml runner to generate xml reports and path to store xml files") + parser.add_argument("-p", "--pthreads", dest="num_threads", + help="The number of threads used", default=1) + + results = parser.parse_args() testResultLogFile = None - if options.result is not None: - testResultLogFile = options.result + if results.result is not None: + testResultLogFile = results.result testCaseLogFile = None - if options.testcaselog is not None: - testCaseLogFile = options.testcaselog - deploy = deployDataCenter.deployDataCenters(options.config) - if options.load: + if results.testcaselog is not None: + testCaseLogFile = results.testcaselog + deploy = deployDataCenter.deployDataCenters(results.config) + if results.load: deploy.loadCfg() else: deploy.deploy() format = "text" xmlDir = None - if options.xmlrunner is not None: - xmlDir = options.xmlrunner + if results.xmlrunner is not None: + xmlDir = results.xmlrunner format = "xml" - if options.testCaseFolder is None: - if options.module is None: + if results.testCaseFolder is None: + if results.module is None: parser.print_usage() exit(1) else: - engine = TestCaseExecuteEngine.TestCaseExecuteEngine(deploy.testClient, testCaseLogFile, testResultLogFile, format, xmlDir) - engine.loadTestsFromFile(options.module) + if results.num_threads == 1: + engine = TestCaseExecuteEngine.TestCaseExecuteEngine(deploy.testClient, testCaseLogFile, testResultLogFile, format, xmlDir) + else: + engine = ThreadedTestCaseExecuteEngine.TestCaseExecuteEngine(deploy.testClient, testCaseLogFile, testResultLogFile, format, xmlDir,results.num_threads) + + engine.loadTestsFromFile(results.module) engine.run() else: - engine = TestCaseExecuteEngine.TestCaseExecuteEngine(deploy.testClient, testCaseLogFile, testResultLogFile, format, xmlDir) - engine.loadTestsFromDir(options.testCaseFolder) + if results.num_threads == 1: + engine = TestCaseExecuteEngine.TestCaseExecuteEngine(deploy.testClient, testCaseLogFile, testResultLogFile, format, xmlDir) + else: + engine = ThreadedTestCaseExecuteEngine.TestCaseExecuteEngine(deploy.testClient, testCaseLogFile, testResultLogFile, format, xmlDir, results.num_threads) + + engine.loadTestsFromDir(results.testCaseFolder) engine.run()
