Just a few things, didn't have time to look over the whole thing in
detail...
On 03/28/2013 01:21 AM, Lucas Meneghel Rodrigues wrote:
Since autotest has been helping to test the Linux virt
technologies, why not let them help to test autotest?
This is an early preview of a test suite designed
to do functional regression testing of autotest
using vms, as well as execute both the unittests
and the pylint checkers on it, and very basic
initial verification of the autotest server features.
It wouldn't be possible without the many cleanups
in static checking run made during the past couple
of weeks, now we're to a point where autotest master
can give useful static checking reports.
Hopefully this code will be refactored, and coverage
expanded.
Changes from v1:
* Fix stupid mistake from one of the rebases I've
made (client ip acquisition was made too early,
resulting in a VMIPAddressMissingError right
in the beginning of the test).
Signed-off-by: Lucas Meneghel Rodrigues<[email protected]>
---
tests/autotest_regression.py | 163 ++++++++++++++++++++++++++++++++++++++
tests/cfg/autotest_regression.cfg | 13 +++
2 files changed, 176 insertions(+)
create mode 100644 tests/autotest_regression.py
create mode 100644 tests/cfg/autotest_regression.cfg
diff --git a/tests/autotest_regression.py b/tests/autotest_regression.py
new file mode 100644
index 0000000..a78820a
--- /dev/null
+++ b/tests/autotest_regression.py
@@ -0,0 +1,163 @@
+import logging
+from autotest.client.shared import error
+from virttest import aexpect, utils_misc
+
+
[email protected]_aware
+def run_autotest_regression(test, params, env):
+ """
+ Autotest regression test:
+
+ Use Virtual Machines to test autotest.
+
+ 1) Clone the given guest OS (only Linux) image twice.
+ 2) Boot 2 VMs (autotest_server_vm and autotest_client_vm)
+ 4) Install the autotest server in the server vm
+ 5) Run the unittests
+ 6) Run the pylint checker
+ 7) Run a simple client sleeptest
+ 8) Run a simple server sleeptest
+ 9) Register the client vm in the autotest server
+ 10) Schedule a simple job sleeptest in the client. Wait for client reboot.
+ 11) If any of these steps have failed, fail the test and report the error
+
+ @param test: virt test object
+ @param params: Dictionary with the test parameters
+ @param env: Dictionary with test environment.
+ """
+ step_failures = []
+ autotest_repo = params['autotest_repo']
+ autotest_branch = params['autotest_branch']
+ autotest_commit = params['autotest_commit']
+ autotest_install_timeout = int(params.get('autotest_install_timeout',
1800))
+ unittests_run_timeout = int(params.get('unittests_run_timeout', 1800))
+ pylint_run_timeout = int(params.get('pylint_run_timeout', 1800))
+ vm_names = params["vms"].split()
+ server_name = vm_names[0]
+ client_name = vm_names[1]
+ vm_server = env.get_vm(server_name)
+ vm_server.verify_alive()
+ vm_client = env.get_vm(client_name)
+ vm_client.verify_alive()
+
+ timeout = float(params.get("login_timeout", 240))
+ session_server = vm_server.wait_for_login(timeout=timeout)
+ session_client = vm_client.wait_for_login(timeout=timeout)
+ client_ip = vm_client.get_address()
+
+ session_server.cmd("yum install -y passwd pylint")
Maybe my idea of the test context is off, but this package install looks
out-of-place here. Shouldn't that be handled by some other aspect/area?
Also, I thought there was some package tool for this. What if someone
wants to run this on non-RPM based server/client?
+
+ step1 = "autotest-server-install"
+ try:
+ # Download the install script and execute it
+ download_cmd = ("wget https://raw.github.com/autotest/autotest/master";
+ "/contrib/install-autotest-server.sh")
+ session_server.cmd(download_cmd)
+ permission_cmd = ("chmod +x install-autotest-server.sh")
+ session_server.cmd(permission_cmd)
+ install_cmd = ("./install-autotest-server.sh -u Aut0t3st -d Aut0t3st "
+ "-g %s -b %s" % (autotest_repo, autotest_branch))
+ if autotest_commit:
+ install_cmd += " -c %s" % autotest_commit
+ session_server.cmd(install_cmd, timeout=autotest_install_timeout)
+ except aexpect.ShellCmdError, e:
+ for line in e.output.splitlines():
+ logging.error(line)
+ step_failures.append(step1)
+
+ top_commit = None
+ try:
+ session_server.cmd("test -d /usr/local/autotest/.git")
+ session_server.cmd("cd /usr/local/autotest")
+ top_commit = session_server.cmd("echo `git log -n 1
--pretty=format:%H`")
+ top_commit = top_commit.strip()
+ logging.info("Autotest top commit for repo %s, branch %s: %s",
+ autotest_repo, autotest_branch, top_commit)
+ except aexpect.ShellCmdError, e:
+ for line in e.output.splitlines():
+ logging.error(line)
+
+ if top_commit is not None:
+ step2 = "unittests"
+ try:
+ session_server.cmd("cd /usr/local/autotest")
+ session_server.cmd("utils/unittest_suite.py --full",
+ timeout=unittests_run_timeout)
+ except aexpect.ShellCmdError, e:
+ for line in e.output.splitlines():
+ logging.error(line)
+ step_failures.append(step2)
+
+ step3 = "pylint"
+ try:
+ session_server.cmd("cd /usr/local/autotest")
+ session_server.cmd("utils/check_patch.py --full --yes",
+ timeout=pylint_run_timeout)
+ except aexpect.ShellCmdError, e:
+ for line in e.output.splitlines():
+ logging.error(line)
+ step_failures.append(step3)
+
+ step4 = "client_run"
+ try:
+ session_server.cmd("cd /usr/local/autotest/client")
+ session_server.cmd("./autotest-local run sleeptest",
+ timeout=pylint_run_timeout)
+ session_server.cmd("rm -rf results/default")
+ except aexpect.ShellCmdError, e:
+ for line in e.output.splitlines():
+ logging.error(line)
+ step_failures.append(step4)
+
+ step5 = "server_run"
+ try:
+ session_client.cmd("iptables -F")
+ session_server.cmd("cd /usr/local/autotest")
+ session_server.cmd("server/autotest-remote -m %s -c "
+ "client/tests/sleeptest/control" % client_ip,
+ timeout=pylint_run_timeout)
+ session_server.cmd("rm -rf results-*")
+ except aexpect.ShellCmdError, e:
+ for line in e.output.splitlines():
+ logging.error(line)
+ step_failures.append(step5)
+
+ step6 = "registering_client_cli"
+ try:
+ session_server.cmd("/usr/local/autotest/cli/autotest-rpc-client "
+ "label create -t vm")
+ session_server.cmd("/usr/local/autotest/cli/autotest-rpc-client "
+ "host create -t vm %s" % client_ip)
+ except aexpect.ShellCmdError, e:
+ for line in e.output.splitlines():
+ logging.error(line)
+ step_failures.append(step6)
+
+ step7 = "running_job_cli"
+ try:
+ session_client.cmd("iptables -F")
+ session_server.cmd("/usr/local/autotest/cli/autotest-rpc-client "
+ "job create --test sleeptest -m %s" % client_ip)
+ except aexpect.ShellCmdError, e:
+ for line in e.output.splitlines():
+ logging.error(line)
+ step_failures.append(step7)
+
+ if not utils_misc.wait_for(vm_client.is_dead, timeout=300):
+ step_failures.append(step7)
+
+ def report_version():
+ if top_commit is not None:
+ logging.info("Autotest git repo: %s", autotest_repo)
+ logging.info("Autotest git branch: %s", autotest_repo)
+ logging.info("Autotest top commit: %s", top_commit)
+
+ if step_failures:
+ logging.error("The autotest regression testing failed")
+ report_version()
+ raise error.TestFail("The autotest regression testing had the "
+ "following steps failed: %s" % step_failures)
+ else:
+ logging.info("The autotest regression testing passed")
+ report_version()
+
IIRC, the default aexpect sessions don't auto-close. You might
double-check that you're not leaving aexpect servers hanging when this
code exits or raises an exception.
--
Chris Evich, RHCA, RHCE, RHCDS, RHCSS
Quality Assurance Engineer
e-mail: cevich + `@' + redhat.com o: 1-888-RED-HAT1 x44214
_______________________________________________
Autotest-kernel mailing list
[email protected]
https://www.redhat.com/mailman/listinfo/autotest-kernel
