Since autotest has been helping to test the Linux virt technologies, why not let them help to test autotest?
This is an early preview of a test suite designed to do functional regression testing of autotest using vms, as well as execute both the unittests and the pylint checkers on it, and very basic initial verification of the autotest server features. It wouldn't be possible without the many cleanups in static checking run made during the past couple of weeks, now we're to a point where autotest master can give useful static checking reports. Hopefully this code will be refactored, and coverage expanded. Changes from v1: * Fix stupid mistake from one of the rebases I've made (client ip acquisition was made too early, resulting in a VMIPAddressMissingError right in the beginning of the test). Signed-off-by: Lucas Meneghel Rodrigues <[email protected]> --- tests/autotest_regression.py | 163 ++++++++++++++++++++++++++++++++++++++ tests/cfg/autotest_regression.cfg | 13 +++ 2 files changed, 176 insertions(+) create mode 100644 tests/autotest_regression.py create mode 100644 tests/cfg/autotest_regression.cfg diff --git a/tests/autotest_regression.py b/tests/autotest_regression.py new file mode 100644 index 0000000..a78820a --- /dev/null +++ b/tests/autotest_regression.py @@ -0,0 +1,163 @@ +import logging +from autotest.client.shared import error +from virttest import aexpect, utils_misc + + [email protected]_aware +def run_autotest_regression(test, params, env): + """ + Autotest regression test: + + Use Virtual Machines to test autotest. + + 1) Clone the given guest OS (only Linux) image twice. + 2) Boot 2 VMs (autotest_server_vm and autotest_client_vm) + 4) Install the autotest server in the server vm + 5) Run the unittests + 6) Run the pylint checker + 7) Run a simple client sleeptest + 8) Run a simple server sleeptest + 9) Register the client vm in the autotest server + 10) Schedule a simple job sleeptest in the client. Wait for client reboot. + 11) If any of these steps have failed, fail the test and report the error + + @param test: virt test object + @param params: Dictionary with the test parameters + @param env: Dictionary with test environment. + """ + step_failures = [] + autotest_repo = params['autotest_repo'] + autotest_branch = params['autotest_branch'] + autotest_commit = params['autotest_commit'] + autotest_install_timeout = int(params.get('autotest_install_timeout', 1800)) + unittests_run_timeout = int(params.get('unittests_run_timeout', 1800)) + pylint_run_timeout = int(params.get('pylint_run_timeout', 1800)) + vm_names = params["vms"].split() + server_name = vm_names[0] + client_name = vm_names[1] + vm_server = env.get_vm(server_name) + vm_server.verify_alive() + vm_client = env.get_vm(client_name) + vm_client.verify_alive() + + timeout = float(params.get("login_timeout", 240)) + session_server = vm_server.wait_for_login(timeout=timeout) + session_client = vm_client.wait_for_login(timeout=timeout) + client_ip = vm_client.get_address() + + session_server.cmd("yum install -y passwd pylint") + + step1 = "autotest-server-install" + try: + # Download the install script and execute it + download_cmd = ("wget https://raw.github.com/autotest/autotest/master"; + "/contrib/install-autotest-server.sh") + session_server.cmd(download_cmd) + permission_cmd = ("chmod +x install-autotest-server.sh") + session_server.cmd(permission_cmd) + install_cmd = ("./install-autotest-server.sh -u Aut0t3st -d Aut0t3st " + "-g %s -b %s" % (autotest_repo, autotest_branch)) + if autotest_commit: + install_cmd += " -c %s" % autotest_commit + session_server.cmd(install_cmd, timeout=autotest_install_timeout) + except aexpect.ShellCmdError, e: + for line in e.output.splitlines(): + logging.error(line) + step_failures.append(step1) + + top_commit = None + try: + session_server.cmd("test -d /usr/local/autotest/.git") + session_server.cmd("cd /usr/local/autotest") + top_commit = session_server.cmd("echo `git log -n 1 --pretty=format:%H`") + top_commit = top_commit.strip() + logging.info("Autotest top commit for repo %s, branch %s: %s", + autotest_repo, autotest_branch, top_commit) + except aexpect.ShellCmdError, e: + for line in e.output.splitlines(): + logging.error(line) + + if top_commit is not None: + step2 = "unittests" + try: + session_server.cmd("cd /usr/local/autotest") + session_server.cmd("utils/unittest_suite.py --full", + timeout=unittests_run_timeout) + except aexpect.ShellCmdError, e: + for line in e.output.splitlines(): + logging.error(line) + step_failures.append(step2) + + step3 = "pylint" + try: + session_server.cmd("cd /usr/local/autotest") + session_server.cmd("utils/check_patch.py --full --yes", + timeout=pylint_run_timeout) + except aexpect.ShellCmdError, e: + for line in e.output.splitlines(): + logging.error(line) + step_failures.append(step3) + + step4 = "client_run" + try: + session_server.cmd("cd /usr/local/autotest/client") + session_server.cmd("./autotest-local run sleeptest", + timeout=pylint_run_timeout) + session_server.cmd("rm -rf results/default") + except aexpect.ShellCmdError, e: + for line in e.output.splitlines(): + logging.error(line) + step_failures.append(step4) + + step5 = "server_run" + try: + session_client.cmd("iptables -F") + session_server.cmd("cd /usr/local/autotest") + session_server.cmd("server/autotest-remote -m %s -c " + "client/tests/sleeptest/control" % client_ip, + timeout=pylint_run_timeout) + session_server.cmd("rm -rf results-*") + except aexpect.ShellCmdError, e: + for line in e.output.splitlines(): + logging.error(line) + step_failures.append(step5) + + step6 = "registering_client_cli" + try: + session_server.cmd("/usr/local/autotest/cli/autotest-rpc-client " + "label create -t vm") + session_server.cmd("/usr/local/autotest/cli/autotest-rpc-client " + "host create -t vm %s" % client_ip) + except aexpect.ShellCmdError, e: + for line in e.output.splitlines(): + logging.error(line) + step_failures.append(step6) + + step7 = "running_job_cli" + try: + session_client.cmd("iptables -F") + session_server.cmd("/usr/local/autotest/cli/autotest-rpc-client " + "job create --test sleeptest -m %s" % client_ip) + except aexpect.ShellCmdError, e: + for line in e.output.splitlines(): + logging.error(line) + step_failures.append(step7) + + if not utils_misc.wait_for(vm_client.is_dead, timeout=300): + step_failures.append(step7) + + def report_version(): + if top_commit is not None: + logging.info("Autotest git repo: %s", autotest_repo) + logging.info("Autotest git branch: %s", autotest_repo) + logging.info("Autotest top commit: %s", top_commit) + + if step_failures: + logging.error("The autotest regression testing failed") + report_version() + raise error.TestFail("The autotest regression testing had the " + "following steps failed: %s" % step_failures) + else: + logging.info("The autotest regression testing passed") + report_version() + diff --git a/tests/cfg/autotest_regression.cfg b/tests/cfg/autotest_regression.cfg new file mode 100644 index 0000000..e811dd9 --- /dev/null +++ b/tests/cfg/autotest_regression.cfg @@ -0,0 +1,13 @@ +- autotest_regression: install setup image_copy unattended_install.cdrom + virt_test_type = qemu libvirt + master_images_clone = image1 + type = autotest_regression + restart_vm = yes + kill_vm_on_error = yes + login_timeout = 240 + vms = "autotest_server_vm autotest_client_vm" + autotest_install_timeout = 1800 + autotest_repo = git://github.com/autotest/autotest.git + autotest_branch = next + autotest_commit = + -- 1.8.1.4 _______________________________________________ Virt-test-devel mailing list [email protected] https://www.redhat.com/mailman/listinfo/virt-test-devel
