Re: [Freeipa-devel] [PATCH 0553] CI tests: improve log collecting in tests

[Martin Babinsky]

On 07/25/2016 06:24 PM, Martin Basti wrote:



On 25.07.2016 18:02, Martin Babinsky wrote:

On 07/22/2016 06:13 PM, Martin Basti wrote:



On 20.07.2016 17:41, Martin Basti wrote:




On 19.07.2016 17:05, Martin Basti wrote:




On 19.07.2016 16:18, Martin Basti wrote:

Patch attached.




self-NACK, my assumptions were wrong, this doesn't work if any of log
files do not exist



updated patches attached

Please note, that in case that any logfile does not exist tar returns
exit code 2, but provides correct output anyway.



Updated patches attached.



NACK:

* Module ipatests.test_integration.tasks
ipatests/test_integration/tasks.py:68: [E1101(no-member),
setup_server_logs_collecting] Instance of 'FedoraPathNamespace' has no
'IPASERVER_CA_INSTALL_LOG' member)
Makefile:137: recipe for target 'lint' failed
make: *** [lint] Error 1


Ooops, I forgot to merge fix

Updated patch attached.

ACK.

Pushed to master: ae623864ee6e5dc2f6d254111c9cdd369fc144a8

--
Martin^3 Babinsky

--
Manage your subscription for the Freeipa-devel mailing list:
https://www.redhat.com/mailman/listinfo/freeipa-devel
Contribute to FreeIPA: http://www.freeipa.org/page/Contribute/Code

Re: [Freeipa-devel] [PATCH 0553] CI tests: improve log collecting in tests

[Martin Basti]

On 25.07.2016 18:02, Martin Babinsky wrote:

On 07/22/2016 06:13 PM, Martin Basti wrote:



On 20.07.2016 17:41, Martin Basti wrote:




On 19.07.2016 17:05, Martin Basti wrote:




On 19.07.2016 16:18, Martin Basti wrote:

Patch attached.




self-NACK, my assumptions were wrong, this doesn't work if any of log
files do not exist



updated patches attached

Please note, that in case that any logfile does not exist tar returns
exit code 2, but provides correct output anyway.



Updated patches attached.



NACK:

* Module ipatests.test_integration.tasks
ipatests/test_integration/tasks.py:68: [E1101(no-member), 
setup_server_logs_collecting] Instance of 'FedoraPathNamespace' has no 
'IPASERVER_CA_INSTALL_LOG' member)

Makefile:137: recipe for target 'lint' failed
make: *** [lint] Error 1


Ooops, I forgot to merge fix

Updated patch attached.
From 2c9e3f05d191c053e94cfeba3822830f6c8f2d0e Mon Sep 17 00:00:00 2001
From: Martin Basti 
Date: Tue, 19 Jul 2016 12:09:29 +0200
Subject: [PATCH 1/2] CI tests: improve log collecting

We should collect as much as possible relevant logs to be able do better
investigation from test automation
---
 ipaplatform/base/paths.py  |  1 +
 ipatests/test_integration/tasks.py | 59 +-
 2 files changed, 46 insertions(+), 14 deletions(-)

diff --git a/ipaplatform/base/paths.py b/ipaplatform/base/paths.py
index 1507ac36da5b40447c951ee608053a09b2db2fc3..579ea70d9f0795443aebb5f120d63c8e5289886c 100644
--- a/ipaplatform/base/paths.py
+++ b/ipaplatform/base/paths.py
@@ -300,6 +300,7 @@ class BasePathNamespace(object):
 SLAPD_INSTANCE_ACCESS_LOG_TEMPLATE = "/var/log/dirsrv/slapd-%s/access"
 SLAPD_INSTANCE_ERROR_LOG_TEMPLATE = "/var/log/dirsrv/slapd-%s/errors"
 VAR_LOG_HTTPD_DIR = "/var/log/httpd"
+VAR_LOG_HTTPD_ERROR = "/var/log/httpd/error_log"
 IPABACKUP_LOG = "/var/log/ipabackup.log"
 IPACLIENT_INSTALL_LOG = "/var/log/ipaclient-install.log"
 IPACLIENT_UNINSTALL_LOG = "/var/log/ipaclient-uninstall.log"
diff --git a/ipatests/test_integration/tasks.py b/ipatests/test_integration/tasks.py
index cee6c15ccdc260f6ebe4cdebbc2cf6ec84931f27..df5fddb4a439dcd3dc73cee8fe4902686ed7b33c 100644
--- a/ipatests/test_integration/tasks.py
+++ b/ipatests/test_integration/tasks.py
@@ -47,6 +47,48 @@ from ipalib.constants import DOMAIN_LEVEL_0
 log = log_mgr.get_logger(__name__)
 
 
+def setup_server_logs_collecting(host):
+"""
+This function setup logs to be collected on host. We should collect all
+possible logs that may be helpful to debug IPA server
+"""
+# dirsrv logs
+inst = host.domain.realm.replace('.', '-')
+host.collect_log(paths.SLAPD_INSTANCE_ERROR_LOG_TEMPLATE % inst)
+host.collect_log(paths.SLAPD_INSTANCE_ACCESS_LOG_TEMPLATE % inst)
+
+# IPA install logs
+host.collect_log(paths.IPASERVER_INSTALL_LOG)
+host.collect_log(paths.IPACLIENT_INSTALL_LOG)
+host.collect_log(paths.IPAREPLICA_INSTALL_LOG)
+host.collect_log(paths.IPAREPLICA_CONNCHECK_LOG)
+host.collect_log(paths.IPAREPLICA_CA_INSTALL_LOG)
+host.collect_log(paths.IPACLIENT_INSTALL_LOG)
+host.collect_log(paths.IPASERVER_KRA_INSTALL_LOG)
+host.collect_log(paths.IPA_CUSTODIA_AUDIT_LOG)
+
+# IPA uninstall logs
+host.collect_log(paths.IPASERVER_KRA_UNINSTALL_LOG)
+host.collect_log(paths.IPACLIENT_UNINSTALL_LOG)
+host.collect_log(paths.IPASERVER_KRA_UNINSTALL_LOG)
+
+# IPA backup and restore logs
+host.collect_log(paths.IPARESTORE_LOG)
+host.collect_log(paths.IPABACKUP_LOG)
+
+# kerberos related logs
+host.collect_log(paths.KADMIND_LOG)
+
+# httpd logs
+host.collect_log(paths.VAR_LOG_HTTPD_ERROR)
+
+# dogtag logs
+host.collect_log(os.path.join(paths.VAR_LOG_PKI_DIR))
+
+# SSSD debugging must be set after client is installed (function
+# setup_sssd_debugging)
+
+
 def check_arguments_are(slice, instanceof):
 """
 :param: slice - tuple of integers denoting the beginning and the end
@@ -234,12 +276,7 @@ def install_master(host, setup_dns=True, setup_kra=False, extra_args=(),
domain_level=None):
 if domain_level is None:
 domain_level = host.config.domain_level
-host.collect_log(paths.IPASERVER_INSTALL_LOG)
-host.collect_log(paths.IPACLIENT_INSTALL_LOG)
-inst = host.domain.realm.replace('.', '-')
-host.collect_log(paths.SLAPD_INSTANCE_ERROR_LOG_TEMPLATE % inst)
-host.collect_log(paths.SLAPD_INSTANCE_ACCESS_LOG_TEMPLATE % inst)
-
+setup_server_logs_collecting(host)
 apply_common_fixes(host)
 fix_apache_semaphores(host)
 
@@ -321,8 +358,7 @@ def install_replica(master, replica, setup_ca=True, setup_dns=False,
 if domain_level is None:
 domain_level = domainlevel(master)
 apply_common_fixes(replica)
-replica.collect_log(paths.IPAREPLICA_INSTALL_LOG)
-replica.collect_log(paths.IPAREPLICA_CONNCHECK_LOG)
+

Re: [Freeipa-devel] [PATCH 0553] CI tests: improve log collecting in tests

[Martin Babinsky]

On 07/22/2016 06:13 PM, Martin Basti wrote:



On 20.07.2016 17:41, Martin Basti wrote:




On 19.07.2016 17:05, Martin Basti wrote:




On 19.07.2016 16:18, Martin Basti wrote:

Patch attached.




self-NACK, my assumptions were wrong, this doesn't work if any of log
files do not exist



updated patches attached

Please note, that in case that any logfile does not exist tar returns
exit code 2, but provides correct output anyway.



Updated patches attached.



NACK:

* Module ipatests.test_integration.tasks
ipatests/test_integration/tasks.py:68: [E1101(no-member), 
setup_server_logs_collecting] Instance of 'FedoraPathNamespace' has no 
'IPASERVER_CA_INSTALL_LOG' member)

Makefile:137: recipe for target 'lint' failed
make: *** [lint] Error 1

--
Martin^3 Babinsky

--
Manage your subscription for the Freeipa-devel mailing list:
https://www.redhat.com/mailman/listinfo/freeipa-devel
Contribute to FreeIPA: http://www.freeipa.org/page/Contribute/Code

Re: [Freeipa-devel] [PATCH 0553] CI tests: improve log collecting in tests

[Martin Basti]

On 20.07.2016 17:41, Martin Basti wrote:




On 19.07.2016 17:05, Martin Basti wrote:




On 19.07.2016 16:18, Martin Basti wrote:

Patch attached.



self-NACK, my assumptions were wrong, this doesn't work if any of log 
files do not exist




updated patches attached

Please note, that in case that any logfile does not exist tar returns 
exit code 2, but provides correct output anyway.




Updated patches attached.
From 6a32476465ab642cc4f6c8edf79e14a3353216f7 Mon Sep 17 00:00:00 2001
From: Martin Basti 
Date: Wed, 20 Jul 2016 17:35:32 +0200
Subject: [PATCH 2/2] CI tests: fix SSSD log collecting

Wildcard '*' has not been working for log collecting. I just set
the whole SSSD log directory to be collected. tar utility is able to
archive whole directories.
---
 ipatests/pytest_plugins/integration.py | 5 +++--
 ipatests/test_integration/tasks.py | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/ipatests/pytest_plugins/integration.py b/ipatests/pytest_plugins/integration.py
index a191d848669111211cc1a13f412e3f777eae565e..63d0c9943901233633fd60271eb4778924297d16 100644
--- a/ipatests/pytest_plugins/integration.py
+++ b/ipatests/pytest_plugins/integration.py
@@ -91,8 +91,9 @@ def collect_logs(name, logs_dict, logfile_dir=None, beakerlib_plugin=None):
 log.info('Collecting logs from: %s', host.hostname)
 
 # Tar up the logs on the remote server
-cmd = host.run_command(['tar', 'cJv'] + logs, log_stdout=False,
-   raiseonerr=False)
+cmd = host.run_command(
+['tar', '-c',  '--ignore-failed-read', '-J', '-v'] + logs,
+log_stdout=False, raiseonerr=False)
 if cmd.returncode:
 log.warning('Could not collect all requested logs')
 
diff --git a/ipatests/test_integration/tasks.py b/ipatests/test_integration/tasks.py
index be9febe3402841f23472e7a09b8f97d1da8f1430..c38cbd2da2a21f712418e125d6e4e466d79ff75d 100644
--- a/ipatests/test_integration/tasks.py
+++ b/ipatests/test_integration/tasks.py
@@ -560,7 +560,7 @@ def setup_sssd_debugging(host):
   paths.SSSD_CONF],
  raiseonerr=False)
 
-host.collect_log(os.path.join(paths.VAR_LOG_SSSD_DIR, '*'))
+host.collect_log(os.path.join(paths.VAR_LOG_SSSD_DIR))
 
 # Clear the cache and restart SSSD
 clear_sssd_cache(host)
-- 
2.5.5

From a6a8e6ec8e048564d50685590480dd3ed271e966 Mon Sep 17 00:00:00 2001
From: Martin Basti 
Date: Tue, 19 Jul 2016 12:09:29 +0200
Subject: [PATCH 1/2] CI tests: improve log collecting

We should collect as much as possible relevant logs to be able do better
investigation from test automation
---
 ipaplatform/base/paths.py  |  1 +
 ipatests/test_integration/tasks.py | 60 +-
 2 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/ipaplatform/base/paths.py b/ipaplatform/base/paths.py
index 1507ac36da5b40447c951ee608053a09b2db2fc3..579ea70d9f0795443aebb5f120d63c8e5289886c 100644
--- a/ipaplatform/base/paths.py
+++ b/ipaplatform/base/paths.py
@@ -300,6 +300,7 @@ class BasePathNamespace(object):
 SLAPD_INSTANCE_ACCESS_LOG_TEMPLATE = "/var/log/dirsrv/slapd-%s/access"
 SLAPD_INSTANCE_ERROR_LOG_TEMPLATE = "/var/log/dirsrv/slapd-%s/errors"
 VAR_LOG_HTTPD_DIR = "/var/log/httpd"
+VAR_LOG_HTTPD_ERROR = "/var/log/httpd/error_log"
 IPABACKUP_LOG = "/var/log/ipabackup.log"
 IPACLIENT_INSTALL_LOG = "/var/log/ipaclient-install.log"
 IPACLIENT_UNINSTALL_LOG = "/var/log/ipaclient-uninstall.log"
diff --git a/ipatests/test_integration/tasks.py b/ipatests/test_integration/tasks.py
index cee6c15ccdc260f6ebe4cdebbc2cf6ec84931f27..be9febe3402841f23472e7a09b8f97d1da8f1430 100644
--- a/ipatests/test_integration/tasks.py
+++ b/ipatests/test_integration/tasks.py
@@ -47,6 +47,49 @@ from ipalib.constants import DOMAIN_LEVEL_0
 log = log_mgr.get_logger(__name__)
 
 
+def setup_server_logs_collecting(host):
+"""
+This function setup logs to be collected on host. We should collect all
+possible logs that may be helpful to debug IPA server
+"""
+# dirsrv logs
+inst = host.domain.realm.replace('.', '-')
+host.collect_log(paths.SLAPD_INSTANCE_ERROR_LOG_TEMPLATE % inst)
+host.collect_log(paths.SLAPD_INSTANCE_ACCESS_LOG_TEMPLATE % inst)
+
+# IPA install logs
+host.collect_log(paths.IPASERVER_INSTALL_LOG)
+host.collect_log(paths.IPACLIENT_INSTALL_LOG)
+host.collect_log(paths.IPAREPLICA_INSTALL_LOG)
+host.collect_log(paths.IPAREPLICA_CONNCHECK_LOG)
+host.collect_log(paths.IPAREPLICA_CA_INSTALL_LOG)
+host.collect_log(paths.IPACLIENT_INSTALL_LOG)
+host.collect_log(paths.IPASERVER_KRA_INSTALL_LOG)
+host.collect_log(paths.IPASERVER_CA_INSTALL_LOG)
+host.collect_log(paths.IPA_CUSTODIA_AUDIT_LOG)
+
+# IPA uninstall logs
+host.collect_log(paths.IPASERVER_KRA_UNINSTALL_LOG)
+

Re: [Freeipa-devel] [PATCH 0553] CI tests: improve log collecting in tests

[Martin Basti]

On 19.07.2016 16:18, Martin Basti wrote:

Patch attached.



self-NACK, my assumptions were wrong, this doesn't work if any of log 
files do not exist
-- 
Manage your subscription for the Freeipa-devel mailing list:
https://www.redhat.com/mailman/listinfo/freeipa-devel
Contribute to FreeIPA: http://www.freeipa.org/page/Contribute/Code