On 10/15/2014 02:58 PM, Petr Viktorin wrote:
This almost completes the switch to pytest. There are two missing things:
- the details of test results (--with-xunit) are not read correctly by
Jenkins. I have a theory I'm investigating here.
- the beakerlib integration is still not ready


I'll not be available for the rest of the week so I'm sending this
early, in case someone wants to take a look.

I've updated (and rearranged) the patches after some more testing. Both points above are fixed. Individual plugins are broken out; some would be nice to even release independently of IPA. (There is some demand for the BeakerLib plugin; for that I'd only need to break the dependency on ipa_log_manager.)


These depend on my patches 0656-0660.

--
PetrĀ³

From 853f5379eea919bee6dea03889beb8d4cab14075 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Thu, 9 Oct 2014 17:02:25 +0200
Subject: [PATCH] dogtag plugin: Don't use doctest syntax for non-doctest
 examples

https://fedorahosted.org/freeipa/ticket/4610
---
 ipaserver/plugins/dogtag.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/ipaserver/plugins/dogtag.py b/ipaserver/plugins/dogtag.py
index 0e141a45c290b84d65b15b8c2c638577a3a39363..4576c9113b1501f9ab32aef16f8be761e92a9806 100644
--- a/ipaserver/plugins/dogtag.py
+++ b/ipaserver/plugins/dogtag.py
@@ -163,16 +163,16 @@
 1. Reading a serial number from CMS requires conversion from hexadecimal
    by converting it into a Python int or long object, use the int constructor:
 
-   >>> serial_number = int(serial_number, 16)
+        serial_number = int(serial_number, 16)
 
 2. Big integers passed to XMLRPC must be decimal unicode strings
 
-   >>> unicode(serial_number)
+       unicode(serial_number)
 
 3. Big integers received from XMLRPC must be converted back to int or long
    objects from the decimal string representation.
 
-   >>> serial_number = int(serial_number)
+       serial_number = int(serial_number)
 
 Xpath pattern matching on node names:
 -------------------------------------
@@ -202,7 +202,7 @@
 solve the chapter problem above is by using a predicate which says if the node
 name begins with 'chapter' it's a match. Here is how you can do that.
 
-    >>> doc.xpath("//book/*[starts-with(name(), 'chapter')]/section[2]")
+        doc.xpath("//book/*[starts-with(name(), 'chapter')]/section[2]")
 
 The built-in starts-with() returns true if its first argument starts with its
 second argument. Thus the example above says if the node name of the second
@@ -219,10 +219,10 @@
 EXSLT regular expression match() function on the node name. Here is how this is
 done:
 
-    >>> regexpNS = "http://exslt.org/regular-expressions";
-    >>> find = etree.XPath("//book/*[re:match(name(), '^chapter(_\d+)$')]/section[2]",
-    ...                    namespaces={'re':regexpNS}
-    >>> find(doc)
+        regexpNS = "http://exslt.org/regular-expressions";
+        find = etree.XPath("//book/*[re:match(name(), '^chapter(_\d+)$')]/section[2]",
+                           namespaces={'re':regexpNS}
+        find(doc)
 
 What is happening here is that etree.XPath() has returned us an evaluator
 function which we bind to the name 'find'. We've passed it a set of namespaces
-- 
2.1.0

From ee846826561ea598aeb5b54cd56397151fdd2473 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Fri, 10 Oct 2014 16:04:05 +0200
Subject: [PATCH] test_webui: Don't use __init__ for test classes

https://fedorahosted.org/freeipa/ticket/4610
---
 ipatests/test_webui/test_automember.py |  1 +
 ipatests/test_webui/test_cert.py       |  4 ++--
 ipatests/test_webui/test_dns.py        |  4 ++--
 ipatests/test_webui/test_host.py       |  4 ++--
 ipatests/test_webui/test_hostgroup.py  |  6 ++++--
 ipatests/test_webui/test_netgroup.py   |  3 ++-
 ipatests/test_webui/test_trust.py      |  8 ++++----
 ipatests/test_webui/ui_driver.py       | 20 +++++---------------
 8 files changed, 22 insertions(+), 28 deletions(-)

diff --git a/ipatests/test_webui/test_automember.py b/ipatests/test_webui/test_automember.py
index 34710cb6e84ce76ea1930f4fbd17fd1331564b69..0378812ccf7cd9d4415e4ff40009eb2fd509286d 100644
--- a/ipatests/test_webui/test_automember.py
+++ b/ipatests/test_webui/test_automember.py
@@ -93,6 +93,7 @@ def test_rebuild_membership_hosts(self):
         self.init_app()
 
         host_util = host_tasks()
+        host_util.setup(self.driver, self.config)
         domain = self.config.get('ipa_domain')
         host1 = 'web1.%s' % domain
         host2 = 'web2.%s' % domain
diff --git a/ipatests/test_webui/test_cert.py b/ipatests/test_webui/test_cert.py
index 979a51e84437e7c8dae8de5412095c1e719e7db4..ec704eb118e94aeda2ffcb75cb1d1814b97791c3 100644
--- a/ipatests/test_webui/test_cert.py
+++ b/ipatests/test_webui/test_cert.py
@@ -29,8 +29,8 @@
 
 class test_cert(UI_driver):
 
-    def __init__(self, *args, **kwargs):
-        super(test_cert, self).__init__(args, kwargs)
+    def setup(self, *args, **kwargs):
+        super(test_cert, self).setup(*args, **kwargs)
 
         if not self.has_ca():
             self.skip('CA not configured')
diff --git a/ipatests/test_webui/test_dns.py b/ipatests/test_webui/test_dns.py
index baa5bfbb157b30ad9e1445ca3dfc3af1a9573de2..4ad15bd6daffed8926d04f6385d6f72fbd78aded 100644
--- a/ipatests/test_webui/test_dns.py
+++ b/ipatests/test_webui/test_dns.py
@@ -87,8 +87,8 @@
 
 class test_dns(UI_driver):
 
-    def __init__(self, *args, **kwargs):
-        super(test_dns, self).__init__(args, kwargs)
+    def setup(self, *args, **kwargs):
+        super(test_dns, self).setup(*args, **kwargs)
 
         if not self.has_dns():
             self.skip('DNS not configured')
diff --git a/ipatests/test_webui/test_host.py b/ipatests/test_webui/test_host.py
index 4b1d4201225e9630993c7116018d5aae5c61ec14..36814842853e5f0091914688e5892ed4d0a15d8f 100644
--- a/ipatests/test_webui/test_host.py
+++ b/ipatests/test_webui/test_host.py
@@ -34,8 +34,8 @@
 
 class host_tasks(UI_driver):
 
-    def __init__(self, *args, **kwargs):
-        super(host_tasks, self).__init__(args, kwargs)
+    def setup(self, *args, **kwargs):
+        super(host_tasks, self).setup(*args, **kwargs)
         self.prep_data()
         self.prep_data2()
 
diff --git a/ipatests/test_webui/test_hostgroup.py b/ipatests/test_webui/test_hostgroup.py
index b83c0b88c512559280bd89df923e633df59ea7f6..e2dbbf588ede505d8eee9f653bc4d54c440a7adf 100644
--- a/ipatests/test_webui/test_hostgroup.py
+++ b/ipatests/test_webui/test_hostgroup.py
@@ -48,7 +48,8 @@ def test_associations(self):
         Hostgroup associations
         """
         self.init_app()
-        host = host_tasks(self.driver, self.config)
+        host = host_tasks()
+        host.setup(self.driver, self.config)
 
         # prepare
         # -------
@@ -89,7 +90,8 @@ def test_indirect_associations(self):
         Hostgroup indirect associations
         """
         self.init_app()
-        host = host_tasks(self.driver, self.config)
+        host = host_tasks()
+        host.setup(self.driver, self.config)
 
         # add
         # ---
diff --git a/ipatests/test_webui/test_netgroup.py b/ipatests/test_webui/test_netgroup.py
index 55333e54bf8f50114c9871c86647cc363f74ac46..3dfea0dfa20cc69855affcc4d247e872b2414d8e 100644
--- a/ipatests/test_webui/test_netgroup.py
+++ b/ipatests/test_webui/test_netgroup.py
@@ -46,7 +46,8 @@ def test_mod(self):
         Mod: netgroup
         """
         self.init_app()
-        host = host_tasks(self.driver, self.config)
+        host = host_tasks()
+        host.setup(self.driver, self.config)
 
         self.add_record(netgroup.ENTITY, netgroup.DATA2)
         self.add_record(user.ENTITY, user.DATA)
diff --git a/ipatests/test_webui/test_trust.py b/ipatests/test_webui/test_trust.py
index 95e0fedda54722e7d9a7e82e238c10d0a127e4ad..138eabf2ec8f262e600435095f481deba07452c6 100644
--- a/ipatests/test_webui/test_trust.py
+++ b/ipatests/test_webui/test_trust.py
@@ -95,9 +95,8 @@ def get_range_name(self):
 
 class test_trust(trust_tasks):
 
-    def __init__(self, *args, **kwargs):
-        super(test_trust, self).__init__(args, kwargs)
-
+    def setup(self, *args, **kwargs):
+        super(test_trust, self).setup(*args, **kwargs)
         if not self.has_trusts():
             self.skip('Trusts not configured')
 
@@ -121,7 +120,8 @@ def test_range_types(self):
 
         self.init_app()
 
-        r_tasks = range_tasks(self.driver, self.config)
+        r_tasks = range_tasks()
+        r_tasks.setup(self.driver, self.config)
         r_tasks.get_shifts()
         range_add = r_tasks.get_add_data('')
         base_id = range_add[2][2]
diff --git a/ipatests/test_webui/ui_driver.py b/ipatests/test_webui/ui_driver.py
index e706a68e0e55024a835e5be3071cccfeee312a1f..c4647726017c7a802c5c248b4d02ca7029462d4e 100644
--- a/ipatests/test_webui/ui_driver.py
+++ b/ipatests/test_webui/ui_driver.py
@@ -117,12 +117,16 @@ def setup_class(cls):
         if NO_SELENIUM:
             raise nose.SkipTest('Selenium not installed')
 
-    def __init__(self, driver=None, config=None):
+    def setup(self, driver=None, config=None):
         self.request_timeout = 30
         self.driver = driver
         self.config = config
         if not config:
             self.load_config()
+        self.get_driver().maximize_window()
+
+    def teardown(self):
+        self.driver.quit()
 
     def load_config(self):
         """
@@ -161,20 +165,6 @@ def load_config(self):
         if 'type' not in c:
             c['type'] = DEFAULT_TYPE
 
-    def setup(self):
-        """
-        Test setup
-        """
-        if not self.driver:
-            self.driver = self.get_driver()
-            self.driver.maximize_window()
-
-    def teardown(self):
-        """
-        Test clean up
-        """
-        self.driver.quit()
-
     def get_driver(self):
         """
         Get WebDriver according to configuration
-- 
2.1.0

From 30ef4db1b172bc853d381d2b55885a5311c223fa Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Thu, 23 Oct 2014 13:31:45 +0200
Subject: [PATCH] test_ipapython: Use functions instead of classes in test
 generators

pytest's support for Nose-style test generators is not bulletproof;
use a real function to please it.

https://fedorahosted.org/freeipa/ticket/4610
---
 ipatests/test_ipapython/test_ipautil.py | 11 ++++++-----
 ipatests/test_ipapython/test_ssh.py     | 11 +++++------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/ipatests/test_ipapython/test_ipautil.py b/ipatests/test_ipapython/test_ipautil.py
index 04a43990ea99564319560de1b59d28d1e4a5bdb6..c882df2cdddb78e75d61ff01330376937fb6677e 100644
--- a/ipatests/test_ipapython/test_ipautil.py
+++ b/ipatests/test_ipapython/test_ipautil.py
@@ -24,16 +24,17 @@
 
 from ipapython import ipautil
 
-class CheckIPAddress:
-    def __init__(self, addr):
-        self.description = "Test IP address parsing and verification (%s)" % addr
 
-    def __call__(self, addr, words=None, prefixlen=None):
+def make_ipaddress_checker(addr, words=None, prefixlen=None):
+    def check_ipaddress():
         try:
             ip = ipautil.CheckedIPAddress(addr, match_local=False)
             assert ip.words == words and ip.prefixlen == prefixlen
         except:
             assert words is None and prefixlen is None
+    check_ipaddress.description = "Test IP address parsing and verification (%s)" % addr
+    return check_ipaddress
+
 
 def test_ip_address():
     addrs = [
@@ -66,7 +67,7 @@ def test_ip_address():
     ]
 
     for addr in addrs:
-        yield (CheckIPAddress(addr[0]),) + addr
+        yield make_ipaddress_checker(*addr)
 
 
 class TestCIDict(object):
diff --git a/ipatests/test_ipapython/test_ssh.py b/ipatests/test_ipapython/test_ssh.py
index 2640af50d2b475886047cf2d4e53e3b3cfefbb75..db136a898e2c9a8be62eafebed2fd01f02631535 100644
--- a/ipatests/test_ipapython/test_ssh.py
+++ b/ipatests/test_ipapython/test_ssh.py
@@ -25,16 +25,15 @@
 
 from ipapython import ssh
 
-class CheckPublicKey:
-    def __init__(self, pk):
-        self.description = "Test SSH public key parsing (%s)" % repr(pk)
-
-    def __call__(self, pk, out):
+def make_public_key_checker(pk, out):
+    def check_public_key():
         try:
             parsed = ssh.SSHPublicKey(pk)
             assert parsed.openssh() == out
         except Exception, e:
             assert type(e) is out
+    check_public_key.description = "Test SSH public key parsing (%s)" % repr(pk)
+    return check_public_key
 
 def test_public_key_parsing():
     b64 = 'AAAAB3NzaC1yc2EAAAADAQABAAABAQDGAX3xAeLeaJggwTqMjxNwa6XHBUAikXPGMzEpVrlLDCZtv00djsFTBi38PkgxBJVkgRWMrcBsr/35lq7P6w8KGIwA8GI48Z0qBS2NBMJ2u9WQ2hjLN6GdMlo77O0uJY3251p12pCVIS/bHRSq8kHO2No8g7KA9fGGcagPfQH+ee3t7HUkpbQkFTmbPPN++r3V8oVUk5LxbryB3UIIVzNmcSIn3JrXynlvui4MixvrtX6zx+O/bBo68o8/eZD26QrahVbA09fivrn/4h3TM019Eu/c2jOdckfU3cHUV/3Tno5d6JicibyaoDDK7S/yjdn5jhaz8MSEayQvFkZkiF0L'
@@ -73,4 +72,4 @@ def test_public_key_parsing():
     ]
 
     for pk in pks:
-        yield (CheckPublicKey(pk[0]),) + pk
+        yield make_public_key_checker(*pk)
-- 
2.1.0

From 546525fbd37e911f92ba38da1167012e5d5d7a5b Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Thu, 9 Oct 2014 17:03:02 +0200
Subject: [PATCH] Configure pytest to run doctests

The pytest.ini file needs to be in or above the directory py.test is called in.
When in IPA project root, this invocation will find ./ipatests/pytest.ini:
    py.test ipatests/
but these will not (they're equivalent):
    py.test .
    py.test
So pytest.ini must be in the project root.

However, setupttols can't include files outside package directories,
so we also need this file to be under ipatests/

Solve the problem by symlinking ./pytest.ini to ipatests/pytest.ini.

https://fedorahosted.org/freeipa/ticket/4610
---
 ipatests/pytest.ini  | 17 +++++++++++++++++
 ipatests/setup.py.in |  2 ++
 pytest.ini           |  1 +
 3 files changed, 20 insertions(+)
 create mode 120000 pytest.ini

diff --git a/ipatests/pytest.ini b/ipatests/pytest.ini
index d2355d9616a2aea000d14fa27c9b35d0ad5fb353..ed462c37a0f68c243ff06068be1fc481a45ac922 100644
--- a/ipatests/pytest.ini
+++ b/ipatests/pytest.ini
@@ -1,2 +1,19 @@
+# pytest configuration
+
+# This file lives in ipatests/pytest.ini, so it can be included by setup.py,
+# and it's symlinked from the project's root directory, so py.test finds it
+# when called with no arguments.
+
 [pytest]
 python_classes = test_ Test
+addopts = --doctest-modules
+            # Ignore files for doc tests.
+            # TODO: ideally, these should all use __name__=='__main__' guards
+          --ignore=setup.py
+          --ignore=setup-client.py
+          --ignore=checks/check-ra.py
+          --ignore=daemons/ipa-otpd/test.py
+          --ignore=doc/examples/python-api.py
+          --ignore=install/share/copy-schema-to-ca.py
+          --ignore=install/share/wsgi.py
+          --ignore=ipapython/py_default_encoding/setup.py
diff --git a/ipatests/setup.py.in b/ipatests/setup.py.in
index 7f041465c132b6a5ab941ba253c2fb6e011b458e..658a330d8f9d4647828aaeb4abfc175d81269d1c 100644
--- a/ipatests/setup.py.in
+++ b/ipatests/setup.py.in
@@ -66,6 +66,7 @@ def setup_package():
             classifiers=filter(None, CLASSIFIERS.split('\n')),
             package_dir = {'ipatests': ''},
             packages = ["ipatests",
+                        "ipatests.pytest_plugins",
                         "ipatests.test_cmdline",
                         "ipatests.test_install",
                         "ipatests.test_integration",
@@ -78,6 +79,7 @@ def setup_package():
                         "ipatests.test_xmlrpc"],
             scripts=['ipa-run-tests', 'ipa-test-config', 'ipa-test-task'],
             package_data = {
+                'ipatests': ['pytest.ini'],
                 'ipatests.test_install': ['*.update'],
                 'ipatests.test_integration': ['scripts/*'],
                 'ipatests.test_pkcs10': ['*.csr'],
diff --git a/pytest.ini b/pytest.ini
new file mode 120000
index 0000000000000000000000000000000000000000..e87991b0fd75787ca830a213d15c0634af811dff
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1 @@
+ipatests/pytest.ini
\ No newline at end of file
-- 
2.1.0

From a25073339d7dd85276610bd8a96d226beff06a87 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Fri, 10 Oct 2014 09:39:00 +0200
Subject: [PATCH] Declarative tests: Move cleanup to setup_class/teardown_class

https://fedorahosted.org/freeipa/ticket/4610
---
 ipatests/test_xmlrpc/xmlrpc_test.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/ipatests/test_xmlrpc/xmlrpc_test.py b/ipatests/test_xmlrpc/xmlrpc_test.py
index 1f44f7794bdd390153317b08b396ac3dfc09e150..306d66f715b504c51bf3189aee7d406ceddb8c36 100644
--- a/ipatests/test_xmlrpc/xmlrpc_test.py
+++ b/ipatests/test_xmlrpc/xmlrpc_test.py
@@ -252,6 +252,18 @@ class Declarative(XMLRPC_test):
     cleanup_commands = tuple()
     tests = tuple()
 
+    @classmethod
+    def setup_class(cls):
+        super(Declarative, cls).setup_class()
+        for command in cls.cleanup_commands:
+            cls.cleanup(command)
+
+    @classmethod
+    def teardown_class(cls):
+        for command in cls.cleanup_commands:
+            cls.cleanup(command)
+        super(Declarative, cls).teardown_class()
+
     def cleanup_generate(self, stage):
         for (i, command) in enumerate(self.cleanup_commands):
             func = lambda: self.cleanup(command)
@@ -260,15 +272,18 @@ def cleanup_generate(self, stage):
             )
             yield (func,)
 
-    def cleanup(self, command):
+    @classmethod
+    def cleanup(cls, command):
         (cmd, args, options) = command
+        print 'Cleanup:', cmd, args, options
         if cmd not in api.Command:
             raise nose.SkipTest(
                 'cleanup command %r not in api.Command' % cmd
             )
         try:
             api.Command[cmd](*args, **options)
-        except (errors.NotFound, errors.EmptyModlist):
+        except (errors.NotFound, errors.EmptyModlist) as e:
+            print e
             pass
 
     def test_generator(self):
@@ -277,12 +292,6 @@ def test_generator(self):
 
         nose reports each one as a seperate test.
         """
-
-        # Iterate through pre-cleanup:
-        for tup in self.cleanup_generate('pre'):
-            yield tup
-
-        # Iterate through the tests:
         name = self.__class__.__name__
         for (i, test) in enumerate(self.tests):
             if callable(test):
@@ -299,10 +308,6 @@ def test_generator(self):
                 func.description = nice
             yield (func,)
 
-        # Iterate through post-cleanup:
-        for tup in self.cleanup_generate('post'):
-            yield tup
-
     def check(self, nice, desc, command, expected, extra_check=None):
         (cmd, args, options) = command
         options.setdefault('version', self.default_version)
-- 
2.1.0

From eebf25474ada78dd773981f91cc3f30b9bc9c76d Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Fri, 10 Oct 2014 09:47:25 +0200
Subject: [PATCH] Declarative tests: Switch to pytest

Provide a local pytest plugin to generate tests.

The Declarative tests can now only be run with pytest

https://fedorahosted.org/freeipa/ticket/4610
---
 ipatests/pytest.ini                    |  1 +
 ipatests/pytest_plugins/__init__.py    |  0
 ipatests/pytest_plugins/declarative.py | 47 ++++++++++++++++++++++++++++++++++
 ipatests/test_xmlrpc/xmlrpc_test.py    | 26 +++++--------------
 4 files changed, 55 insertions(+), 19 deletions(-)
 create mode 100644 ipatests/pytest_plugins/__init__.py
 create mode 100644 ipatests/pytest_plugins/declarative.py

diff --git a/ipatests/pytest.ini b/ipatests/pytest.ini
index ed462c37a0f68c243ff06068be1fc481a45ac922..d4ff3f00d5899a0ed420ad886937dd97dc8b263b 100644
--- a/ipatests/pytest.ini
+++ b/ipatests/pytest.ini
@@ -7,6 +7,7 @@
 [pytest]
 python_classes = test_ Test
 addopts = --doctest-modules
+          -p ipatests.pytest_plugins.declarative
             # Ignore files for doc tests.
             # TODO: ideally, these should all use __name__=='__main__' guards
           --ignore=setup.py
diff --git a/ipatests/pytest_plugins/__init__.py b/ipatests/pytest_plugins/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/ipatests/pytest_plugins/declarative.py b/ipatests/pytest_plugins/declarative.py
new file mode 100644
index 0000000000000000000000000000000000000000..3f88bb850d4e0fd43e0a8122ac042c86bc1e8a10
--- /dev/null
+++ b/ipatests/pytest_plugins/declarative.py
@@ -0,0 +1,47 @@
+# Authors:
+#   Petr Viktorin <pvikt...@redhat.com>
+#
+# Copyright (C) 2011  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Pytest plugin for Declarative tests"""
+
+
+def pytest_generate_tests(metafunc):
+    """Generates Declarative tests"""
+    if 'declarative_test_definition' in metafunc.fixturenames:
+        name = metafunc.cls.__name__
+        tests = []
+        descriptions = []
+        for i, test in enumerate(metafunc.cls.tests):
+            if callable(test):
+                description = '%d: %s' % (
+                    i,
+                    test.__name__,  # test is not a dict. pylint: disable=E1103
+                )
+            else:
+                description = '%d: %s: %s' % (i,
+                                              test['command'][0],
+                                              test.get('desc', ''))
+                test = dict(test)
+                test['nice'] = description
+            tests.append(test)
+            descriptions.append(description)
+        metafunc.parametrize(
+            ['index', 'declarative_test_definition'],
+            enumerate(tests),
+            ids=descriptions,
+        )
diff --git a/ipatests/test_xmlrpc/xmlrpc_test.py b/ipatests/test_xmlrpc/xmlrpc_test.py
index 306d66f715b504c51bf3189aee7d406ceddb8c36..59b023df936c2d042a319c177d8047a6d460956e 100644
--- a/ipatests/test_xmlrpc/xmlrpc_test.py
+++ b/ipatests/test_xmlrpc/xmlrpc_test.py
@@ -286,27 +286,15 @@ def cleanup(cls, command):
             print e
             pass
 
-    def test_generator(self):
-        """
-        Iterate through tests.
+    def test_command(self, index, declarative_test_definition):
+        """Run an individual test
 
-        nose reports each one as a seperate test.
+        The arguments are provided by the pytest plugin.
         """
-        name = self.__class__.__name__
-        for (i, test) in enumerate(self.tests):
-            if callable(test):
-                func = lambda: test(self)
-                nice = '%s[%d]: call %s: %s' % (
-                    name, i, test.__name__, test.__doc__
-                )
-                func.description = nice
-            else:
-                nice = '%s[%d]: %s: %s' % (
-                    name, i, test['command'][0], test.get('desc', '')
-                )
-                func = lambda: self.check(nice, **test)
-                func.description = nice
-            yield (func,)
+        if callable(declarative_test_definition):
+            declarative_test_definition(self)
+        else:
+            self.check(**declarative_test_definition)
 
     def check(self, nice, desc, command, expected, extra_check=None):
         (cmd, args, options) = command
-- 
2.1.0

From 50dc19584472136e8bef9ddfa4c42ac305229b73 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Fri, 10 Oct 2014 14:56:29 +0200
Subject: [PATCH] Integration tests: Port the ordering plugin to pytest

Ordered integration tests may now be run with pytest.

https://fedorahosted.org/freeipa/ticket/4610
---
 freeipa.spec.in                            |   1 +
 ipatests/order_plugin.py                   | 106 -----------------------------
 ipatests/pytest.ini                        |   1 +
 ipatests/pytest_plugins/ordering.py        |  88 ++++++++++++++++++++++++
 ipatests/test_integration/base.py          |   2 +-
 ipatests/test_integration/test_caless.py   |   2 +-
 ipatests/test_integration/test_ordering.py |  54 +++++++++++++++
 7 files changed, 146 insertions(+), 108 deletions(-)
 delete mode 100644 ipatests/order_plugin.py
 create mode 100644 ipatests/pytest_plugins/ordering.py
 create mode 100644 ipatests/test_integration/test_ordering.py

diff --git a/freeipa.spec.in b/freeipa.spec.in
index 8fcb535e229db4f7a8eaaee3c99b18446eef7f1e..e67aef8f809b9d14aad9ed46498a50838c8bdec5 100644
--- a/freeipa.spec.in
+++ b/freeipa.spec.in
@@ -299,6 +299,7 @@ Requires: %{name}-python = %{version}-%{release}
 Requires: tar
 Requires: xz
 Requires: python-nose
+Requires: pytest >= 2.6
 Requires: python-paste
 Requires: python-coverage
 Requires: python-polib
diff --git a/ipatests/order_plugin.py b/ipatests/order_plugin.py
deleted file mode 100644
index 7b114a56783a6b3f2e34498567a61c2668ab5c08..0000000000000000000000000000000000000000
--- a/ipatests/order_plugin.py
+++ /dev/null
@@ -1,106 +0,0 @@
-# Authors:
-#   Petr Viktorin <pvikt...@redhat.com>
-#
-# Copyright (C) 2013  Red Hat
-# see file 'COPYING' for use and warranty information
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-"""A Nose plugin that allows ordered test cases"""
-
-import os
-import unittest
-import inspect
-
-from nose.plugins import Plugin
-import nose.loader
-import nose.util
-
-
-def ordered(cls):
-    """Decorator that marks a test class as ordered
-
-    Methods within the marked class will be executed in definition order
-    (or more strictly, in ordered by the line number where they're defined).
-
-    Subclasses of unittest.TestCase can not be ordered.
-
-    Generator methods will not be ordered by this plugin.
-    """
-    cls._order_plugin__ordered = True
-    assert not isinstance(cls, unittest.TestCase), (
-        "A unittest.TestCase may not be ordered.")
-    return cls
-
-
-class OrderTests(Plugin):
-    name = 'ordered-tests'
-
-    def options(self, parser, env=os.environ):
-        super(OrderTests, self).options(parser, env=env)
-
-    def configure(self, options, conf):
-        super(OrderTests, self).configure(options, conf)
-        if not self.enabled:
-            return
-
-    def loadTestsFromTestClass(self, cls):
-        """Sort methods of ordered test cases by co_firstlineno"""
-        if not getattr(cls, '_order_plugin__ordered', False):
-            return
-        loader = nose.loader.TestLoader()
-
-        def wanted(attr):
-            item = getattr(cls, attr, None)
-            if not inspect.ismethod(item):
-                return False
-            if nose.util.isgenerator(item.im_func):
-                return False
-            return loader.selector.wantMethod(item)
-
-        def sort_with_respect_to_overriding(func):
-            """
-            Sorts the methods in respect with the parent classes.
-
-            The methods are sorted with respect to the inheritance chain,
-            methods that were defined in the same class are sorted by the line
-            number on which they were defined.
-            """
-
-            # Check each *ordered* class in MRO for definition of func method
-            for i, parent_class in enumerate(reversed(cls.mro())):
-                if getattr(parent_class, '_order_plugin__ordered', False):
-                    method = getattr(parent_class, func.__name__, None)
-                    if method:
-                        # This sorts methods as tuples  (position of the class
-                        # in the inheritance chain, position of the method
-                        # within that class)
-                        return 0, i, method.func_code.co_firstlineno
-
-            # Weird case fallback
-            # Method name not in any of the classes in MRO, run it last
-            return 1, func.func_code.co_firstlineno
-
-        methods = [getattr(cls, case) for case in dir(cls) if wanted(case)]
-        methods.sort(key=sort_with_respect_to_overriding)
-        cases = [loader.makeTest(m, cls) for m in methods]
-        return cases
-
-    def wantMethod(self, method):
-        """Hide non-TestCase methods from the normal loader"""
-        im_class = getattr(method, 'im_class', None)
-        if im_class and getattr(im_class, '_order_plugin__ordered', False):
-            if nose.util.isgenerator(method.im_func):
-                return True
-            return False
diff --git a/ipatests/pytest.ini b/ipatests/pytest.ini
index d4ff3f00d5899a0ed420ad886937dd97dc8b263b..fbd6558abfff4ac7df858836780a1f98e9219aa6 100644
--- a/ipatests/pytest.ini
+++ b/ipatests/pytest.ini
@@ -8,6 +8,7 @@
 python_classes = test_ Test
 addopts = --doctest-modules
           -p ipatests.pytest_plugins.declarative
+          -p ipatests.pytest_plugins.ordering
             # Ignore files for doc tests.
             # TODO: ideally, these should all use __name__=='__main__' guards
           --ignore=setup.py
diff --git a/ipatests/pytest_plugins/ordering.py b/ipatests/pytest_plugins/ordering.py
new file mode 100644
index 0000000000000000000000000000000000000000..3af496a88c3072372d17c743ad4b84f7a75c5ff0
--- /dev/null
+++ b/ipatests/pytest_plugins/ordering.py
@@ -0,0 +1,88 @@
+# Authors:
+#   Petr Viktorin <pvikt...@redhat.com>
+#
+# Copyright (C) 2014  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Pytest plugin for IPA
+
+Adds support for the @pytest.mark.source_order decorator which,
+when applied to a class, runs the test methods in source order.
+
+See test_ordering for an example.
+"""
+
+import unittest
+
+import pytest
+
+
+def ordered(cls):
+    """Decorator that marks a test class as ordered
+
+    Methods within the marked class will be executed in definition order
+    (or more strictly, in ordered by the line number where they're defined).
+
+    Subclasses of unittest.TestCase can not be ordered.
+
+    Generator methods will not be ordered by this plugin.
+    """
+    cls._order_plugin__ordered = True
+    assert not isinstance(cls, unittest.TestCase), (
+        "A unittest.TestCase may not be ordered.")
+    cls = pytest.mark.source_order(cls)
+    return cls
+
+
+def decorate_items(items):
+    node_indexes = {}
+    for index, item in enumerate(items):
+        try:
+            func = item.function
+        except AttributeError:
+            yield (index, ), item
+            continue
+
+        key = (index, )
+        for node in reversed(item.listchain()):
+            # Find the corresponding class
+            if isinstance(node, pytest.Class):
+                cls = node.cls
+            else:
+                continue
+            if getattr(cls, '_order_plugin__ordered', False):
+                node_index = node_indexes.setdefault(node, index)
+                # Find first occurence of the method in class hierarchy
+                for i, parent_class in enumerate(reversed(cls.mro())):
+                    if getattr(parent_class, '_order_plugin__ordered', False):
+                        method = getattr(parent_class, func.__name__, None)
+                        if method:
+                            # Sort methods as tuples  (position of the class
+                            # in the inheritance chain, position of the method
+                            # within that class)
+                            key = (node_index, 0,
+                                   i, method.func_code.co_firstlineno, node)
+                            break
+                else:
+                    # Weird case fallback
+                    # Method name not in any of the classes in MRO, run it last
+                    key = node_index, 1, func.func_code.co_firstlineno, node
+                break
+        yield key, item
+
+
+def pytest_collection_modifyitems(session, config, items):
+    items[:] = [item for i, item in sorted(decorate_items(items))]
diff --git a/ipatests/test_integration/base.py b/ipatests/test_integration/base.py
index a24a577d61b17a46979bdb324faece8e5942d312..b07eab0995db11ddf963d289324a8717655b0ea9 100644
--- a/ipatests/test_integration/base.py
+++ b/ipatests/test_integration/base.py
@@ -24,7 +24,7 @@
 from ipapython.ipa_log_manager import log_mgr
 from ipatests.test_integration.config import get_global_config
 from ipatests.test_integration import tasks
-from ipatests.order_plugin import ordered
+from ipatests.pytest_plugins.ordering import ordered
 
 log = log_mgr.get_logger(__name__)
 
diff --git a/ipatests/test_integration/test_caless.py b/ipatests/test_integration/test_caless.py
index b2af4f7fe89d18bb81b12021e3a6e2eb8d47b40c..19a425e157c831011312702159321bd81a41cda7 100644
--- a/ipatests/test_integration/test_caless.py
+++ b/ipatests/test_integration/test_caless.py
@@ -31,7 +31,7 @@
 from ipapython.dn import DN
 from ipatests.test_integration.base import IntegrationTest
 from ipatests.test_integration import tasks
-from ipatests.order_plugin import ordered
+from ipatests.pytest_plugins.ordering import ordered
 
 _DEFAULT = object()
 
diff --git a/ipatests/test_integration/test_ordering.py b/ipatests/test_integration/test_ordering.py
new file mode 100644
index 0000000000000000000000000000000000000000..ee224901c5ee05355f148cf132c15427adfee497
--- /dev/null
+++ b/ipatests/test_integration/test_ordering.py
@@ -0,0 +1,54 @@
+# Authors:
+#   Petr Viktorin <pvikt...@redhat.com>
+#
+# Copyright (C) 2014  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the ordering of tests
+
+IPA integration tests, marked with `@ordered`, require tests to be run
+in a specific order:
+- Base classes first
+- Within a class, test methods are ordered according to source line
+"""
+
+from ipatests.pytest_plugins.ordering import ordered
+
+
+@ordered
+class TestBase(object):
+    @classmethod
+    def setup_class(cls):
+        cls.value = 'unchanged'
+
+    def test_d_first(self):
+        type(self).value = 'changed once'
+
+
+class TestChild(TestBase):
+    def test_b_third(self):
+        assert type(self).value == 'changed twice'
+        type(self).value = 'changed thrice'
+
+    def test_a_fourth(self):
+        assert type(self).value == 'changed thrice'
+
+
+def test_c_second(self):
+    assert type(self).value == 'changed once'
+    type(self).value = 'changed twice'
+TestBase.test_c_second = test_c_second
+del test_c_second
-- 
2.1.0

From cc6df86bb01d7fc2c8d0b08f6388c43fd53f6f63 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Fri, 10 Oct 2014 17:15:52 +0200
Subject: [PATCH] Switch make-test to pytest

The unused capability to run on multiple Python versions is removed,
and needed arguments are now listed in pytest.ini,
leaving just a simple call to the actual test runner.

https://fedorahosted.org/freeipa/ticket/4610
---
 make-test | 63 +++------------------------------------------------------------
 1 file changed, 3 insertions(+), 60 deletions(-)

diff --git a/make-test b/make-test
index 2a48ab18181fe313b413b8c64f300533215380bc..df4e1ad92721fc7c0d2d20f4dae87d4b47cd8ccd 100755
--- a/make-test
+++ b/make-test
@@ -1,61 +1,4 @@
-#!/usr/bin/python2
+#! /bin/bash
 
-"""
-Run IPA unit tests under multiple versions of Python (if present).
-"""
-
-import sys
-import optparse
-import os
-from os import path
-from subprocess import call
-
-versions = ('2.4', '2.5', '2.6', '2.7')
-python = '/usr/bin/python'
-nose = '/usr/bin/nosetests'
-ran = []
-fail = []
-
-cmd = [
-    nose,
-    '-v',
-    '--with-doctest',
-    '--doctest-tests',
-    '--exclude=plugins',
-]
-cmd += sys.argv[1:]
-
-
-# This must be set so ipalib.api gets initialized property for tests:
-os.environ['IPA_UNIT_TEST_MODE'] = 'cli_test'
-
-# Add in-tree client binaries to PATH
-os.environ['PATH'] = './ipa-client:' + os.environ['PATH']
-
-if not path.isfile(nose):
-    print 'ERROR: need %r' % nose
-    sys.exit(100)
-for v in versions:
-    pver = python + v
-    if not path.isfile(pver):
-        continue
-    command = [pver] + cmd
-    print ' '.join(cmd)
-    if 0 != call(cmd):
-        fail.append(pver)
-    ran.append(pver)
-
-
-print '=' * 70
-for pver in ran:
-    if pver in fail:
-        print 'FAILED under %r' % pver
-    else:
-        print 'passed under %r' % pver
-print ''
-if fail:
-    print '** FAIL **'
-    sys.exit(1)
-else:
-    print '** pass **'
-    sys.exit(0)
+set -ex
+IPA_UNIT_TEST_MODE=cli_test py.test "$@"
-- 
2.1.0

From a571c1174b49d703387c6ec4ab6f84a4233bf1f9 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Mon, 13 Oct 2014 13:27:18 +0200
Subject: [PATCH] Add local pytest plugin for --with-xunit and --logging-level

The --with-xunit option ihas the same behavior as in nosetests:
it's an alias for pytest's --junitxml=nosetests.py

The --logging-level option enables direct IPA logging to stdout.

https://fedorahosted.org/freeipa/ticket/4610
---
 ipatests/pytest.ini                    |  2 +
 ipatests/pytest_plugins/nose_compat.py | 73 ++++++++++++++++++++++++++++++++++
 2 files changed, 75 insertions(+)
 create mode 100644 ipatests/pytest_plugins/nose_compat.py

diff --git a/ipatests/pytest.ini b/ipatests/pytest.ini
index fbd6558abfff4ac7df858836780a1f98e9219aa6..32dc858b006f3245587277dea21aea6e6d6be938 100644
--- a/ipatests/pytest.ini
+++ b/ipatests/pytest.ini
@@ -7,6 +7,8 @@
 [pytest]
 python_classes = test_ Test
 addopts = --doctest-modules
+          --junit-prefix ipa
+          -p ipatests.pytest_plugins.nose_compat
           -p ipatests.pytest_plugins.declarative
           -p ipatests.pytest_plugins.ordering
             # Ignore files for doc tests.
diff --git a/ipatests/pytest_plugins/nose_compat.py b/ipatests/pytest_plugins/nose_compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..67eb6e074a7ed3aea5f5ab7f02a3d2a38c30f627
--- /dev/null
+++ b/ipatests/pytest_plugins/nose_compat.py
@@ -0,0 +1,73 @@
+# Authors:
+#   Petr Viktorin <pvikt...@redhat.com>
+#
+# Copyright (C) 2014  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Provides command-line options for very limited Nose compatibility"""
+
+import os
+import sys
+import logging
+
+from ipapython.ipa_log_manager import log_mgr
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("IPA nosetests compatibility shim")
+
+    group.addoption('--with-xunit', action="store_const",
+           dest="xmlpath", metavar="path",  default=None,
+           const=os.environ.get('IPATEST_XUNIT_PATH', './nosetests.xml'),
+           help="create junit-xml style report file at $IPATEST_XUNIT_PATH,"
+                "or nosetests.xml by default")
+
+    group.addoption('--logging-level', action="store",
+           dest="logging_level", metavar="level", default='CRITICAL',
+           help="level for logging to stderr. "
+                "Bypasses pytest logging redirection."
+                "May be used to show progress of long-running tests.")
+
+
+def pytest_configure(config):
+    if config.getoption('logging_level'):
+        # Forward IPA logging to a normal Python logger. Nose's logcapture plugin
+        # can't work with IPA-managed loggers
+        class LogHandler(logging.Handler):
+            name = 'forwarding log handler'
+            logger = logging.getLogger('IPA')
+
+            def emit(self, record):
+                capture = config.pluginmanager.getplugin('capturemanager')
+                orig_stdout, orig_stderr = sys.stdout, sys.stderr
+                if capture:
+                    capture._capturing.suspend_capturing()
+                sys.stderr.write(self.format(record))
+                sys.stderr.write('\n')
+                if capture:
+                    capture._capturing.resume_capturing()
+                sys.stdout, sys.stderr = orig_stdout, orig_stderr
+
+        log_mgr.configure(
+            {
+                'default_level': config.getoption('logging_level'),
+                'handlers': [{'log_handler': LogHandler(),
+                            'format': '[%(name)s] %(message)s',
+                            'level': 'debug'},
+                            {'level': 'debug',
+                            'name': 'real_stderr',
+                            'stream': sys.stderr}]},
+            configure_state='tests')
-- 
2.1.0

From 902a3f3a3c12b30a2a27bcb57558aa2a948eff39 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Mon, 13 Oct 2014 14:34:53 +0200
Subject: [PATCH] Switch ipa-run-tests to pytest

https://fedorahosted.org/freeipa/ticket/4610
---
 ipatests/ipa-run-tests | 56 +++++++++-----------------------------------------
 1 file changed, 10 insertions(+), 46 deletions(-)

diff --git a/ipatests/ipa-run-tests b/ipatests/ipa-run-tests
index 7e3270b41e28d9c75473cfdb3f8c36ef016509a0..53fa7b3218e7aad54f065c5042d3a4bef017fcd5 100755
--- a/ipatests/ipa-run-tests
+++ b/ipatests/ipa-run-tests
@@ -20,62 +20,26 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Nose wrapper for running an installed (not in-tree) IPA test suite
+"""Pytest wrapper for running an installed (not in-tree) IPA test suite
 
-Any command-line arguments are passed directly to Nose.
-Note that any relative paths given will be based on the ipatests module's path
+Any command-line arguments are passed directly to py.test.
+The current directory is changed to the locaition of the ipatests package,
+so any relative paths given will be based on the ipatests module's path
 """
 
-import sys
 import os
-from os import path
-import logging
+import sys
 
-import nose
+import pytest
 
-from ipapython.ipa_log_manager import log_mgr
 import ipatests
-from ipatests.beakerlib_plugin import BeakerLibPlugin
-from ipatests.order_plugin import OrderTests
-
-cmd = [
-    sys.argv[0],
-    '--with-doctest',
-    '--doctest-tests',
-    '--with-ordered-tests',
-    '--exclude=plugins',
-    '--nologcapture',
-    '--logging-filter=-paramiko',
-    '--where', os.path.dirname(ipatests.__file__),
-]
-cmd += sys.argv[1:]
-
 
 # This must be set so ipalib.api gets initialized property for tests:
 os.environ['IPA_UNIT_TEST_MODE'] = 'cli_test'
 
+# This is set to store --with-xunit report in an accessible place:
+os.environ['IPATEST_XUNIT_PATH'] = os.path.join(os.getcwd(), 'nosetests.xml')
 
-# Forward IPA logging to a normal Python logger. Nose's logcapture plugin
-# can't work with IPA-managed loggers
-class LogHandler(logging.Handler):
-    name = 'forwarding log handler'
-    logger = logging.getLogger('IPA')
+os.chdir(os.path.dirname(ipatests.__file__))
 
-    def emit(self, record):
-        self.logger.log(record.levelno, self.format(record))
-
-if 'console' in log_mgr.handlers:
-    log_mgr.remove_handler('console')
-log_mgr.configure(
-    {
-        'default_level': 'DEBUG',
-        'handlers': [{'log_handler': LogHandler(),
-                      'format': '[%(name)s] %(message)s',
-                      'level': 'debug'},
-                     {'level': 'debug',
-                      'name': 'console',
-                      'stream': sys.stderr}]},
-    configure_state='tests')
-
-
-nose.main(argv=cmd, addplugins=[BeakerLibPlugin(), OrderTests()])
+sys.exit(pytest.main(sys.argv[1:]))
-- 
2.1.0

From 1f4300fa894b06b2dc4e3b9cf8995f303986c2d5 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Thu, 23 Oct 2014 19:17:09 +0200
Subject: [PATCH] Switch integration testing config to a fixture

The hack of storing the config on the class is left in;
it would be too much work for too little gain at this time.

https://fedorahosted.org/freeipa/ticket/4610
---
 ipatests/pytest.ini                    |  1 +
 ipatests/pytest_plugins/integration.py | 89 ++++++++++++++++++++++++++++++++++
 ipatests/test_integration/base.py      | 64 ++----------------------
 3 files changed, 94 insertions(+), 60 deletions(-)
 create mode 100644 ipatests/pytest_plugins/integration.py

diff --git a/ipatests/pytest.ini b/ipatests/pytest.ini
index 32dc858b006f3245587277dea21aea6e6d6be938..38b0484c012f006f93772d56ad5208deb8395f74 100644
--- a/ipatests/pytest.ini
+++ b/ipatests/pytest.ini
@@ -11,6 +11,7 @@ addopts = --doctest-modules
           -p ipatests.pytest_plugins.nose_compat
           -p ipatests.pytest_plugins.declarative
           -p ipatests.pytest_plugins.ordering
+          -p ipatests.pytest_plugins.integration
             # Ignore files for doc tests.
             # TODO: ideally, these should all use __name__=='__main__' guards
           --ignore=setup.py
diff --git a/ipatests/pytest_plugins/integration.py b/ipatests/pytest_plugins/integration.py
new file mode 100644
index 0000000000000000000000000000000000000000..0b86546e63e09658879db0a3f5c0dce40fb76cc8
--- /dev/null
+++ b/ipatests/pytest_plugins/integration.py
@@ -0,0 +1,89 @@
+# Authors:
+#   Petr Viktorin <pvikt...@redhat.com>
+#
+# Copyright (C) 2011  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Pytest plugin for IPA Integration tests"""
+
+import pytest
+
+from ipatests.test_integration.config import get_global_config
+
+
+@pytest.yield_fixture(scope='class')
+def integration_config(request):
+    cls = request.cls
+
+    def get_resources(resource_container, resource_str, num_needed):
+        if len(resource_container) < num_needed:
+            raise pytest.skip(
+                'Not enough %s available (have %s, need %s)' %
+                (resource_str, len(resource_container), num_needed))
+        return resource_container[:num_needed]
+
+    config = get_global_config()
+    if not config.domains:
+        raise pytest.skip('Integration testing not configured')
+
+    cls.logs_to_collect = {}
+
+    cls.domain = config.domains[0]
+
+    # Check that we have enough resources available
+    cls.master = cls.domain.master
+    cls.replicas = get_resources(cls.domain.replicas, 'replicas',
+                                    cls.num_replicas)
+    cls.clients = get_resources(cls.domain.clients, 'clients',
+                                cls.num_clients)
+    cls.ad_domains = get_resources(config.ad_domains, 'AD domains',
+                                    cls.num_ad_domains)
+
+    # Check that we have all required extra hosts at our disposal
+    available_extra_roles = [role for domain in cls.get_domains()
+                                    for role in domain.extra_roles]
+    missing_extra_roles = list(set(cls.required_extra_roles) -
+                                    set(available_extra_roles))
+
+    if missing_extra_roles:
+        raise pytest.skip("Not all required extra hosts available, "
+                          "missing: %s, available: %s"
+                          % (missing_extra_roles,
+                             available_extra_roles))
+
+    for host in cls.get_all_hosts():
+        host.add_log_collector(cls.collect_log)
+        cls.prepare_host(host)
+
+    try:
+        cls.install()
+    except:
+        cls.uninstall()
+        raise
+
+    yield config
+
+    for host in cls.get_all_hosts():
+        host.remove_log_collector(cls.collect_log)
+
+    try:
+        cls.uninstall()
+    finally:
+        del cls.master
+        del cls.replicas
+        del cls.clients
+        del cls.ad_domains
+        del cls.domain
diff --git a/ipatests/test_integration/base.py b/ipatests/test_integration/base.py
index b07eab0995db11ddf963d289324a8717655b0ea9..60b0f02a99e8f104e22fc4c9ca9c96f0109c2769 100644
--- a/ipatests/test_integration/base.py
+++ b/ipatests/test_integration/base.py
@@ -19,10 +19,9 @@
 
 """Base class for FreeIPA integration tests"""
 
-import nose
+import pytest
 
 from ipapython.ipa_log_manager import log_mgr
-from ipatests.test_integration.config import get_global_config
 from ipatests.test_integration import tasks
 from ipatests.pytest_plugins.ordering import ordered
 
@@ -30,6 +29,7 @@
 
 
 @ordered
+@pytest.mark.usefixtures('integration_config')
 class IntegrationTest(object):
     num_replicas = 0
     num_clients = 0
@@ -39,52 +39,7 @@ class IntegrationTest(object):
 
     @classmethod
     def setup_class(cls):
-
-        def get_resources(resource_container, resource_str, num_needed):
-            if len(resource_container) < num_needed:
-                raise nose.SkipTest(
-                    'Not enough %s available (have %s, need %s)' %
-                    (resource_str, len(resource_container), num_needed))
-            return resource_container[:num_needed]
-
-        config = get_global_config()
-        if not config.domains:
-            raise nose.SkipTest('Integration testing not configured')
-
-        cls.logs_to_collect = {}
-
-        cls.domain = config.domains[0]
-
-        # Check that we have enough resources available
-        cls.master = cls.domain.master
-        cls.replicas = get_resources(cls.domain.replicas, 'replicas',
-                                     cls.num_replicas)
-        cls.clients = get_resources(cls.domain.clients, 'clients',
-                                    cls.num_clients)
-        cls.ad_domains = get_resources(config.ad_domains, 'AD domains',
-                                       cls.num_ad_domains)
-
-        # Check that we have all required extra hosts at our disposal
-        available_extra_roles = [role for domain in cls.get_domains()
-                                        for role in domain.extra_roles]
-        missing_extra_roles = list(set(cls.required_extra_roles) -
-                                     set(available_extra_roles))
-
-        if missing_extra_roles:
-            raise nose.SkipTest("Not all required extra hosts available, "
-                                "missing: %s, available: %s"
-                                % (missing_extra_roles,
-                                   available_extra_roles))
-
-        for host in cls.get_all_hosts():
-            host.add_log_collector(cls.collect_log)
-            cls.prepare_host(host)
-
-        try:
-            cls.install()
-        except:
-            cls.uninstall()
-            raise
+        pass
 
     @classmethod
     def host_by_role(cls, role):
@@ -116,20 +71,9 @@ def install(cls):
         else:
             tasks.install_topo(cls.topology,
                                cls.master, cls.replicas, cls.clients)
-
     @classmethod
     def teardown_class(cls):
-        for host in cls.get_all_hosts():
-            host.remove_log_collector(cls.collect_log)
-
-        try:
-            cls.uninstall()
-        finally:
-            del cls.master
-            del cls.replicas
-            del cls.clients
-            del cls.ad_domains
-            del cls.domain
+        pass
 
     @classmethod
     def uninstall(cls):
-- 
2.1.0

From 7c9b598bc010ddceff9a78cebd28fd2a6cfef598 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pvikt...@redhat.com>
Date: Thu, 23 Oct 2014 20:56:15 +0200
Subject: [PATCH] Integration tests: Port the BeakerLib plugin and log
 collection to pytest

Move the IPA-specific log collection out of the Beakerlib plugin.
Add the --logfile-dir option to tests and ipa-test-task, so that logs
can be collected even if BeakerLib is not used.

https://fedorahosted.org/freeipa/ticket/4610
---
 ipatests/beakerlib_plugin.py           | 285 ---------------------------------
 ipatests/ipa-test-task                 |  10 +-
 ipatests/pytest.ini                    |   1 +
 ipatests/pytest_plugins/beakerlib.py   | 234 +++++++++++++++++++++++++++
 ipatests/pytest_plugins/integration.py | 137 +++++++++++++++-
 ipatests/test_integration/base.py      |   6 +-
 6 files changed, 378 insertions(+), 295 deletions(-)
 delete mode 100644 ipatests/beakerlib_plugin.py
 create mode 100644 ipatests/pytest_plugins/beakerlib.py

diff --git a/ipatests/beakerlib_plugin.py b/ipatests/beakerlib_plugin.py
deleted file mode 100644
index 1f7811a687dc648e6f841ec576172d23a6ec30ab..0000000000000000000000000000000000000000
--- a/ipatests/beakerlib_plugin.py
+++ /dev/null
@@ -1,285 +0,0 @@
-# Authors:
-#   Petr Viktorin <pvikt...@redhat.com>
-#
-# Copyright (C) 2013  Red Hat
-# see file 'COPYING' for use and warranty information
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-"""A Nose plugin that integrates with BeakerLib"""
-
-import os
-import sys
-import subprocess
-import traceback
-import logging
-import tempfile
-import re
-
-import nose
-from nose.plugins import Plugin
-
-from ipapython import ipautil
-from ipaplatform.paths import paths
-from ipapython.ipa_log_manager import log_mgr
-
-LINK_RE = re.compile(r'https?://[^\s]+')
-
-
-class BeakerLibLogHandler(logging.Handler):
-    def __init__(self, beakerlib_command):
-        super(BeakerLibLogHandler, self).__init__()
-        self.beakerlib_command = beakerlib_command
-
-    def emit(self, record):
-        command = {
-            'DEBUG': 'rlLogDebug',
-            'INFO': 'rlLogInfo',
-            'WARNING': 'rlLogWarning',
-            'ERROR': 'rlLogError',
-            'CRITICAL': 'rlLogFatal',
-        }.get(record.levelname, 'rlLog')
-        self.beakerlib_command([command, self.format(record)])
-
-
-class BeakerLibProcess(object):
-    def __init__(self, env=os.environ):
-        self.log = log_mgr.get_logger(self)
-
-        if 'BEAKERLIB' not in env:
-            raise RuntimeError('$BEAKERLIB not set, cannot use BeakerLib')
-
-        self.env = env
-        # Set up the Bash process
-        self.bash = subprocess.Popen(['bash'],
-                                     stdin=subprocess.PIPE,
-                                     stdout=open(paths.DEV_NULL, 'w'),
-                                     stderr=open(paths.DEV_NULL, 'w'))
-        source_path = os.path.join(self.env['BEAKERLIB'], 'beakerlib.sh')
-        self.run_beakerlib_command(['.', source_path])
-
-        # _in_class_setup is set when we are in setup_class, so logs can be
-        # collected just before the first test starts
-        self._in_class_setup = False
-
-        # Redirect logging to our own handlers
-        self.setup_log_handler(BeakerLibLogHandler(self.run_beakerlib_command))
-
-    def setup_log_handler(self, handler):
-        log_mgr.configure(
-            {
-                'default_level': 'DEBUG',
-                'handlers': [{'log_handler': handler,
-                              'format': '[%(name)s] %(message)s',
-                              'level': 'info'}]},
-            configure_state='beakerlib_plugin')
-
-    def run_beakerlib_command(self, cmd):
-        """Given a command as a Popen-style list, run it in the Bash process"""
-        if not self.bash:
-            return
-        for word in cmd:
-            self.bash.stdin.write(ipautil.shell_quote(word))
-            self.bash.stdin.write(' ')
-        self.bash.stdin.write('\n')
-        self.bash.stdin.flush()
-        assert self.bash.returncode is None, "BeakerLib Bash process exited"
-
-    def log_links(self, docstring):
-        for match in LINK_RE.finditer(docstring or ''):
-            self.log.info('Link: %s', match.group())
-
-    def end(self):
-        """End the Bash process"""
-        self.run_beakerlib_command(['exit'])
-        bash = self.bash
-        self.bash = None
-        bash.communicate()
-
-    def collect_logs(self, logs_to_collect):
-        """Collect specified logs"""
-        for host, logs in logs_to_collect.items():
-            self.log.info('Collecting logs from: %s', host.hostname)
-
-            # Tar up the logs on the remote server
-            cmd = host.run_command(['tar', 'cJv'] + logs, log_stdout=False,
-                                    raiseonerr=False)
-            if cmd.returncode:
-                self.log.warn('Could not collect all requested logs')
-                return
-
-            # Copy and unpack on the local side
-            topdirname = tempfile.mkdtemp()
-            dirname = os.path.join(topdirname, host.hostname)
-            os.mkdir(dirname)
-            tarname = os.path.join(dirname, 'logs.tar.xz')
-            with open(tarname, 'w') as f:
-                f.write(cmd.stdout_text)
-            ipautil.run(['tar', 'xJvf', 'logs.tar.xz'], cwd=dirname)
-            os.unlink(tarname)
-
-            # Use BeakerLib's rlFileSubmit on the indifidual files
-            # The resulting submitted filename will be
-            # $HOSTNAME-$FILENAME (with '/' replaced by '-')
-            self.run_beakerlib_command(['pushd', topdirname])
-            for dirpath, dirnames, filenames in os.walk(topdirname):
-                for filename in filenames:
-                    fullname = os.path.relpath(
-                        os.path.join(dirpath, filename), topdirname)
-                    self.log.debug('Submitting file: %s', fullname)
-                    self.run_beakerlib_command(['rlFileSubmit', fullname])
-            self.run_beakerlib_command(['popd'])
-
-            # The BeakerLib process runs asynchronously, let it clean up
-            # after it's done with the directory
-            self.run_beakerlib_command(['rm', '-rvf', topdirname])
-
-        logs_to_collect.clear()
-
-    def log_exception(self, err=None):
-        """Log an exception
-
-        err is a 3-tuple as returned from sys.exc_info(); if not given,
-        sys.exc_info() is used.
-        """
-        if err is None:
-            err = sys.exc_info()
-        message = ''.join(traceback.format_exception(*err)).rstrip()
-        self.run_beakerlib_command(['rlLogError', message])
-
-
-class BeakerLibPlugin(Plugin):
-    """A Nose plugin that integrates with BeakerLib"""
-    # Since BeakerLib is a Bash library, we need to run it in Bash.
-    # The plugin maintains a Bash process and feeds it with commands
-    # on events like test start/end, logging, etc.
-    # See nose.plugins.base.IPluginInterface for Nose plugin interface docs
-    name = 'beakerlib'
-
-    def __init__(self):
-        super(BeakerLibPlugin, self).__init__()
-        self.log = log_mgr.get_logger(self)
-        self._in_class_setup = False
-
-    def options(self, parser, env=os.environ):
-        super(BeakerLibPlugin, self).options(parser, env=env)
-        self.env = env
-        self.parser = parser
-
-    def configure(self, options, conf):
-        super(BeakerLibPlugin, self).configure(options, conf)
-        if not self.enabled:
-            return
-
-        if 'BEAKERLIB' not in self.env:
-            self.parser.error(
-                '$BEAKERLIB not set, cannot use --with-beakerlib')
-
-        self.process = BeakerLibProcess(env=self.env)
-
-    def run_beakerlib_command(self, cmd):
-        """Given a command as a Popen-style list, run it in the Bash process"""
-        self.process.run_beakerlib_command(cmd)
-
-    def report(self, stream):
-        self.process.end()
-
-    def log_exception(self, err):
-        self.process.log_exception(err)
-
-    def log_links(self, docstring):
-        self.process.log_links(docstring)
-
-    def startContext(self, context):
-        """Start a test context (module, class)
-
-        For test classes, this starts a BeakerLib phase
-        """
-        if not isinstance(context, type):
-            return
-        try:
-            docstring = context.__doc__
-            caption = docstring.strip().partition('\n')[0]
-        except AttributeError:
-            docstring = ''
-            caption = 'Nose class (no docstring)'
-        phase_name = "%s-%s: %s" % (context.__module__.replace('.', '-'),
-                                    context.__name__, caption)
-        self.run_beakerlib_command(['rlPhaseStart', 'FAIL', phase_name])
-        self._in_class_setup = True
-        self.log_links(docstring)
-
-    def stopContext(self, context):
-        """End a test context"""
-        if not isinstance(context, type):
-            return
-        self.collect_logs(context)
-        self.run_beakerlib_command(['rlPhaseEnd'])
-
-    def startTest(self, test):
-        """Start a test phase"""
-        if self._in_class_setup:
-            self.collect_logs(test.context)
-        self.log.info('Running test: %s', test.id())
-        caption = test.shortDescription()
-        if not caption:
-            caption = 'Nose method (no docstring)'
-        phase_name = test.id().replace('.', '-')
-        method = test
-        while hasattr(method, 'test'):
-            method = method.test
-        argument = getattr(method, 'test_argument', None)
-        if argument:
-            phase_name += '-%s' % re.sub('[^-a-zA-Z0-9]+', '_', str(argument))
-        phase_name += ": %s" % caption
-        self.run_beakerlib_command(['rlPhaseStart', 'FAIL', phase_name])
-
-        while hasattr(test, 'test'):
-            # Un-wrap Nose test cases to get at the actual test method
-            test = test.test
-        self.log_links(getattr(test, '__doc__', ''))
-
-    def stopTest(self, test):
-        """End a test phase"""
-        self.collect_logs(test.context)
-        self.run_beakerlib_command(['rlPhaseEnd'])
-
-    def addSuccess(self, test):
-        self.run_beakerlib_command(['rlPass', 'Test succeeded'])
-
-    def addError(self, test, err):
-        if issubclass(err[0], nose.SkipTest):
-            # Log skipped test.
-            # Unfortunately we only get to see this if the built-in skip
-            # plugin is disabled (--no-skip)
-            self.run_beakerlib_command(['rlPass', 'Test skipped: %s' % err[1]])
-        else:
-            self.log_exception(err)
-            self.run_beakerlib_command(
-                ['rlFail', 'Test failed: unhandled exception'])
-        self.collect_logs(test.context)
-
-    def addFailure(self, test, err):
-        self.log_exception(err)
-        self.run_beakerlib_command(['rlFail', 'Test failed'])
-
-    def collect_logs(self, test):
-        """Collect logs specified in test's logs_to_collect attribute
-        """
-        try:
-            logs_to_collect = test.logs_to_collect
-        except AttributeError:
-            self.log.debug('No logs to collect')
-        else:
-            self.process.collect_logs(logs_to_collect)
diff --git a/ipatests/ipa-test-task b/ipatests/ipa-test-task
index 02a04b6ab3fabbefec0ea6b33e2d37b955e529f3..612974549363277fdfe101734cf9defc59c99ab8 100755
--- a/ipatests/ipa-test-task
+++ b/ipatests/ipa-test-task
@@ -28,7 +28,8 @@ from ipapython.ipa_log_manager import log_mgr, standard_logging_setup
 from ipatests.test_integration import config
 from ipatests.test_integration import tasks
 from ipatests.test_integration.host import Host
-from ipatests.beakerlib_plugin import BeakerLibProcess
+from ipatests.pytest_plugins.beakerlib import BeakerLibProcess
+from ipatests.pytest_plugins.integration import collect_logs
 
 
 log = log_mgr.get_logger(__name__)
@@ -50,6 +51,9 @@ class TaskRunner(object):
                             help="""Issue BeakerLib commands for logging
                                     and log collection""")
 
+        parser.add_argument('--logfile-dir', dest='logfile_dir',
+                            help="""Directory to collect logs in""")
+
         subparsers = parser.add_subparsers(
             metavar='SUBCOMMAND',
             help='The action to perform (* indicates an idempotent operation)')
@@ -282,8 +286,10 @@ class TaskRunner(object):
             raise
         finally:
             if args.with_beakerlib:
+                collect_logs('ipa-test-task', logs_to_collect,
+                             logfile_dir=args.logfile_dir,
+                             beakerlib_plugin=beakerlib_process)
                 beakerlib_process.end()
-                beakerlib_process.collect_logs(logs_to_collect)
             for host in self._prepared_hosts:
                 host.remove_log_collector(self.collect_log)
 
diff --git a/ipatests/pytest.ini b/ipatests/pytest.ini
index 38b0484c012f006f93772d56ad5208deb8395f74..3531482d266758b2c31b9775994b16b92bc23bcf 100644
--- a/ipatests/pytest.ini
+++ b/ipatests/pytest.ini
@@ -12,6 +12,7 @@ addopts = --doctest-modules
           -p ipatests.pytest_plugins.declarative
           -p ipatests.pytest_plugins.ordering
           -p ipatests.pytest_plugins.integration
+          -p ipatests.pytest_plugins.beakerlib
             # Ignore files for doc tests.
             # TODO: ideally, these should all use __name__=='__main__' guards
           --ignore=setup.py
diff --git a/ipatests/pytest_plugins/beakerlib.py b/ipatests/pytest_plugins/beakerlib.py
new file mode 100644
index 0000000000000000000000000000000000000000..45bbb05394bf61de90a8c53627c36bb5f60f1f9d
--- /dev/null
+++ b/ipatests/pytest_plugins/beakerlib.py
@@ -0,0 +1,234 @@
+#!/usr/bin/python2
+# Copyright (C) 2014  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""pytest integration with BeakerLib
+
+Runs a Bash process on the side, and feeds BeakerLib commands to it
+(rlPhaseStart, rlPhaseEnd, rlPass, rlFail, ...)
+
+Other plugins may integrate with this using pytest's
+config.pluginmanager.getplugin('BeakerLibPlugin'). If this is None,
+BeakerLib integration is not active, otherwise the result's
+run_beakerlib_command method can be used to run additional commands.
+
+IPA logging is also redirected to the Bash process.
+"""
+
+import os
+import re
+import logging
+import subprocess
+
+import pytest
+
+from ipapython import ipautil
+from ipapython.ipa_log_manager import log_mgr
+
+
+@pytest.fixture
+def log_files_to_collect():
+    return []
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        '--with-beakerlib', action="store_true",
+        dest="with_beakerlib", default=None,
+        help="Report test results via beakerlib")
+
+
+@pytest.mark.tryfirst
+def pytest_load_initial_conftests(args, early_config, parser):
+    ns = early_config.known_args_namespace
+    if ns.with_beakerlib:
+        if 'BEAKERLIB' not in os.environ:
+            raise exit('$BEAKERLIB not set, cannot use --with-beakerlib')
+
+        plugin = BeakerLibPlugin()
+        pluginmanager = early_config.pluginmanager.register(
+            plugin, 'BeakerLibPlugin')
+
+
+class BeakerLibLogHandler(logging.Handler):
+    def __init__(self, beakerlib_command):
+        super(BeakerLibLogHandler, self).__init__()
+        self.beakerlib_command = beakerlib_command
+
+    def emit(self, record):
+        command = {
+            'DEBUG': 'rlLogDebug',
+            'INFO': 'rlLogInfo',
+            'WARNING': 'rlLogWarning',
+            'ERROR': 'rlLogError',
+            'CRITICAL': 'rlLogFatal',
+        }.get(record.levelname, 'rlLog')
+        self.beakerlib_command([command, self.format(record)])
+
+
+class BeakerLibProcess(object):
+    """Manager of a Bash process that is being fed beakerlib commands
+    """
+    def __init__(self, env=os.environ):
+        self.log = log_mgr.get_logger(self)
+
+        if 'BEAKERLIB' not in env:
+            raise RuntimeError('$BEAKERLIB not set, cannot use BeakerLib')
+
+        self.env = env
+        # Set up the Bash process
+        self.bash = subprocess.Popen(['bash'],
+                                     stdin=subprocess.PIPE,
+                                     stdout=open(os.devnull, 'w'),
+                                     stderr=open(os.devnull, 'w'))
+        source_path = os.path.join(self.env['BEAKERLIB'], 'beakerlib.sh')
+        self.run_beakerlib_command(['.', source_path])
+
+        # Redirect logging to our own handlers
+        self.setup_log_handler(BeakerLibLogHandler(self.run_beakerlib_command))
+
+    def setup_log_handler(self, handler):
+        log_mgr.configure(
+            {
+                'default_level': 'DEBUG',
+                'handlers': [{'log_handler': handler,
+                              'format': '[%(name)s] %(message)s',
+                              'level': 'info'}]},
+            configure_state='beakerlib_plugin')
+
+    def run_beakerlib_command(self, cmd):
+        """Given a command as a Popen-style list, run it in the Bash process"""
+        if not self.bash:
+            return
+        for word in cmd:
+            self.bash.stdin.write(ipautil.shell_quote(word))
+            self.bash.stdin.write(' ')
+        self.bash.stdin.write('\n')
+        self.bash.stdin.flush()
+        assert self.bash.returncode is None, "BeakerLib Bash process exited"
+
+    def log_links(self, docstring):
+        for match in LINK_RE.finditer(docstring or ''):
+            self.log.info('Link: %s', match.group())
+
+    def end(self):
+        """End the Bash process"""
+        self.run_beakerlib_command(['exit'])
+        bash = self.bash
+        self.bash = None
+        bash.communicate()
+
+    def log_exception(self, err=None):
+        """Log an exception
+
+        err is a 3-tuple as returned from sys.exc_info(); if not given,
+        sys.exc_info() is used.
+        """
+        if err is None:
+            err = sys.exc_info()
+        message = ''.join(traceback.format_exception(*err)).rstrip()
+        self.run_beakerlib_command(['rlLogError', message])
+
+
+class BeakerLibPlugin(object):
+    def __init__(self):
+        self.log = log_mgr.get_logger(self)
+
+        self.process = BeakerLibProcess(env=os.environ)
+
+        self._current_item = None
+
+    def run_beakerlib_command(self, cmd):
+        """Given a command as a Popen-style list, run it in the Bash process"""
+        self.process.run_beakerlib_command(cmd)
+
+    def get_item_name(self, item):
+        """Return a "identifier-style" name for the given item
+
+        The name only contains the characters [^a-zA-Z0-9_].
+        """
+        bad_char_re = re.compile('[^a-zA-Z0-9_]')
+        parts = []
+        current = item
+        while current:
+            if isinstance(current, pytest.Module):
+                name = current.name
+                if name.endswith('.py'):
+                    name = name[:-3]
+                name = bad_char_re.sub('-', name)
+                parts.append(name)
+                break
+            if isinstance(current, pytest.Instance):
+                pass
+            else:
+                name = current.name
+                name = bad_char_re.sub('-', name)
+                parts.append(name)
+            current = current.parent
+        return '-'.join(reversed(parts))
+
+    def set_current_item(self, item):
+        """Set the item that is currently being processed
+
+        No-op if the same item is already being processed.
+        Ends the phase for the previous item, if any.
+        """
+        if item != self._current_item:
+            item_name = self.get_item_name(item)
+            if self._current_item:
+                self.run_beakerlib_command(['rlPhaseEnd'])
+            if item:
+                self.run_beakerlib_command(['rlPhaseStart', 'FAIL', item_name])
+            self._current_item = item
+
+    def pytest_collection_modifyitems(self, session, config, items):
+        """Log all collected items at start of test"""
+        self.run_beakerlib_command(['rlLogInfo', 'Collected pytest tests:'])
+        for item in items:
+            self.run_beakerlib_command(['rlLogInfo',
+                                        '  - ' + self.get_item_name(item)])
+
+    def pytest_runtest_setup(self, item):
+        """Log item before running it"""
+        self.set_current_item(item)
+
+    def pytest_runtest_makereport(self, item, call):
+        """Report pass/fail for setup/call/teardown of an item"""
+        self.set_current_item(item)
+        desc = '%s: %s' % (call.when, item)
+
+        if not call.excinfo:
+            self.run_beakerlib_command(['rlPass', 'PASS %s' % desc])
+        else:
+            self.run_beakerlib_command(['rlLogError', call.excinfo.exconly()])
+            short_repr = str(call.excinfo.getrepr(style='short'))
+            self.run_beakerlib_command(['rlLogInfo', short_repr])
+
+            # Give super-detailed traceback for DEBUG=1
+            long_repr = str(call.excinfo.getrepr(
+                showlocals=True, funcargs=True))
+            self.run_beakerlib_command(['rlLogDebug', long_repr])
+
+            if call.excinfo.errisinstance(pytest.skip.Exception):
+                self.run_beakerlib_command(['rlPass', 'SKIP %s' % desc])
+            else:
+                self.run_beakerlib_command(['rlFail', 'FAIL %s' % desc])
+
+    def pytest_unconfigure(self, config):
+        """Clean up and exit"""
+        self.set_current_item(None)
+        self.process.end()
diff --git a/ipatests/pytest_plugins/integration.py b/ipatests/pytest_plugins/integration.py
index 0b86546e63e09658879db0a3f5c0dce40fb76cc8..f6468155ea2f8071bdbb0946fed47848efbfdafa 100644
--- a/ipatests/pytest_plugins/integration.py
+++ b/ipatests/pytest_plugins/integration.py
@@ -19,13 +19,137 @@
 
 """Pytest plugin for IPA Integration tests"""
 
+import os
+import tempfile
+import shutil
+
 import pytest
 
+from ipapython import ipautil
+from ipapython.ipa_log_manager import log_mgr
 from ipatests.test_integration.config import get_global_config
 
 
+log = log_mgr.get_logger(__name__)
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("IPA integration tests")
+
+    group.addoption(
+        '--logfile-dir', dest="logfile_dir", default=None,
+        help="Directory to store integration test logs in.")
+
+
+def collect_test_logs(node, logs_dict, test_config):
+    """Collect logs from a test
+
+    Calls collect_logs
+
+    :param node: The pytest collection node (request.node)
+    :param logs_dict: Mapping of host to list of log filnames to collect
+    :param test_config: Pytest configuration
+    """
+    collect_logs(
+        name=node.nodeid.replace('/', '-').replace('::', '-'),
+        logs_dict=logs_dict,
+        logfile_dir=test_config.getoption('logfile_dir'),
+        beakerlib_plugin=test_config.pluginmanager.getplugin('BeakerLibPlugin'),
+    )
+
+
+def collect_logs(name, logs_dict, logfile_dir=None, beakerlib_plugin=None):
+    """Collect logs from remote hosts
+
+    Calls collect_logs
+
+    :param name: Name under which logs arecollected, e.g. name of the test
+    :param logs_dict: Mapping of host to list of log filnames to collect
+    :param logfile_dir: Directory to log to
+    :param beakerlib_plugin:
+        BeakerLibProcess or BeakerLibPlugin used to collect tests for BeakerLib
+
+    If neither logfile_dir nor beakerlib_plugin is given, no tests are
+    collected.
+    """
+    if logs_dict and (logfile_dir or beakerlib_plugin):
+
+        if logfile_dir:
+            remove_dir = False
+        else:
+            logfile_dir = tempfile.mkdtemp()
+            remove_dir = True
+
+        topdirname = os.path.join(logfile_dir, name)
+
+        for host, logs in logs_dict.items():
+            log.info('Collecting logs from: %s', host.hostname)
+
+            # Tar up the logs on the remote server
+            cmd = host.run_command(['tar', 'cJv'] + logs, log_stdout=False,
+                                   raiseonerr=False)
+            if cmd.returncode:
+                log.warn('Could not collect all requested logs')
+
+            # Unpack on the local side
+            dirname = os.path.join(topdirname, host.hostname)
+            try:
+                os.makedirs(dirname)
+            except OSError:
+                pass
+            tarname = os.path.join(dirname, 'logs.tar.xz')
+            with open(tarname, 'w') as f:
+                f.write(cmd.stdout_text)
+            ipautil.run(['tar', 'xJvf', 'logs.tar.xz'], cwd=dirname,
+                        raiseonerr=False)
+            os.unlink(tarname)
+
+        if beakerlib_plugin:
+            # Use BeakerLib's rlFileSubmit on the indifidual files
+            # The resulting submitted filename will be
+            # $HOSTNAME-$FILENAME (with '/' replaced by '-')
+            beakerlib_plugin.run_beakerlib_command(['pushd', topdirname])
+            try:
+                for dirpath, dirnames, filenames in os.walk(topdirname):
+                    for filename in filenames:
+                        fullname = os.path.relpath(
+                            os.path.join(dirpath, filename), topdirname)
+                        log.debug('Submitting file: %s', fullname)
+                        beakerlib_plugin.run_beakerlib_command(
+                            ['rlFileSubmit', fullname])
+            finally:
+                beakerlib_plugin.run_beakerlib_command(['popd'])
+
+        if remove_dir:
+            if beakerlib_plugin:
+                # The BeakerLib process runs asynchronously, let it clean up
+                # after it's done with the directory
+                beakerlib_plugin.run_beakerlib_command(
+                    ['rm', '-rvf', topdirname])
+            else:
+                shutil.rmtree(topdirname)
+
+        logs_dict.clear()
+
+
+@pytest.fixture(scope='class')
+def class_integration_logs():
+    """Internal fixture providing class-level logs_dict"""
+    return {}
+
+
+@pytest.yield_fixture
+def integration_logs(class_integration_logs, request):
+    """Provides access to test integration logs, and collects after each test
+    """
+    yield class_integration_logs
+    collect_test_logs(request.node, class_integration_logs, request.config)
+
+
 @pytest.yield_fixture(scope='class')
-def integration_config(request):
+def integration_config(request, class_integration_logs):
+    """Integration test Config object
+    """
     cls = request.cls
 
     def get_resources(resource_container, resource_str, num_needed):
@@ -39,7 +163,7 @@ def get_resources(resource_container, resource_str, num_needed):
     if not config.domains:
         raise pytest.skip('Integration testing not configured')
 
-    cls.logs_to_collect = {}
+    cls.logs_to_collect = class_integration_logs
 
     cls.domain = config.domains[0]
 
@@ -64,8 +188,13 @@ def get_resources(resource_container, resource_str, num_needed):
                           % (missing_extra_roles,
                              available_extra_roles))
 
+    def collect_log(host, filename):
+        log.info('Adding %s:%s to list of logs to collect' %
+                 (host.external_hostname, filename))
+        class_integration_logs.setdefault(host, []).append(filename)
+
     for host in cls.get_all_hosts():
-        host.add_log_collector(cls.collect_log)
+        host.add_log_collector(collect_log)
         cls.prepare_host(host)
 
     try:
@@ -79,6 +208,8 @@ def get_resources(resource_container, resource_str, num_needed):
     for host in cls.get_all_hosts():
         host.remove_log_collector(cls.collect_log)
 
+    collect_test_logs(request.node, class_integration_logs, request.config)
+
     try:
         cls.uninstall()
     finally:
diff --git a/ipatests/test_integration/base.py b/ipatests/test_integration/base.py
index 60b0f02a99e8f104e22fc4c9ca9c96f0109c2769..c8b98126e3bb8d8fcd48054fa6b167d3daec6b63 100644
--- a/ipatests/test_integration/base.py
+++ b/ipatests/test_integration/base.py
@@ -30,6 +30,7 @@
 
 @ordered
 @pytest.mark.usefixtures('integration_config')
+@pytest.mark.usefixtures('integration_logs')
 class IntegrationTest(object):
     num_replicas = 0
     num_clients = 0
@@ -83,10 +84,5 @@ def uninstall(cls):
         for client in cls.clients:
             tasks.uninstall_client(client)
 
-    @classmethod
-    def collect_log(cls, host, filename):
-        cls.log.info('Adding %s:%s to list of logs to collect' %
-                     (host.external_hostname, filename))
-        cls.logs_to_collect.setdefault(host, []).append(filename)
 
 IntegrationTest.log = log_mgr.get_logger(IntegrationTest())
-- 
2.1.0

_______________________________________________
Freeipa-devel mailing list
Freeipa-devel@redhat.com
https://www.redhat.com/mailman/listinfo/freeipa-devel

Reply via email to