Updated Branches: refs/heads/trunk 0e7341019 -> d4ab3ab90
Add log statements to our ParamikoSSHClient wrapper. This should make debugging deployment issues easier. Part of LIBCLOUD-414. Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/df6dc611 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/df6dc611 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/df6dc611 Branch: refs/heads/trunk Commit: df6dc6119003c7d6ebe3bf3a237ad4d9d56982d1 Parents: 0e73410 Author: Tomaz Muraus <[email protected]> Authored: Mon Oct 21 22:17:48 2013 +0200 Committer: Tomaz Muraus <[email protected]> Committed: Mon Oct 21 22:17:48 2013 +0200 ---------------------------------------------------------------------- CHANGES | 8 +++++ libcloud/compute/ssh.py | 45 ++++++++++++++++++------- libcloud/test/compute/test_ssh_client.py | 22 +++++++++++-- libcloud/utils/logging.py | 47 +++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/df6dc611/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 8cc1449..fe8db26 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,13 @@ -*- coding: utf-8 -*- +Changes with Apache Libcloud in development + + *) Compute + + - Add log statements to our ParamikoSSHClient wrapper. This should make + debugging deployment issues easier. (LIBCLOUD-414) + [Tomaz Muraus] + Changes with Apache Libcloud 0.14.0-beta1 *) General http://git-wip-us.apache.org/repos/asf/libcloud/blob/df6dc611/libcloud/compute/ssh.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/ssh.py b/libcloud/compute/ssh.py index bc23e4d..d476e87 100644 --- a/libcloud/compute/ssh.py +++ b/libcloud/compute/ssh.py @@ -35,6 +35,8 @@ import logging from os.path import split as psplit from os.path import join as pjoin +from libcloud.utils.logging import ExtraLogFormatter + class BaseSSHClient(object): """ @@ -136,6 +138,18 @@ class BaseSSHClient(object): raise NotImplementedError( 'close not implemented for this ssh client') + def _get_and_setup_logger(self): + logger = logging.getLogger('libcloud.compute.ssh') + path = os.getenv('LIBCLOUD_DEBUG') + + if path: + handler = logging.FileHandler(path) + handler.setFormatter(ExtraLogFormatter()) + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + + return logger + class ParamikoSSHClient(BaseSSHClient): @@ -148,6 +162,7 @@ class ParamikoSSHClient(BaseSSHClient): password, key, timeout) self.client = paramiko.SSHClient() self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.logger = self._get_and_setup_logger() def connect(self): conninfo = {'hostname': self.hostname, @@ -167,10 +182,17 @@ class ParamikoSSHClient(BaseSSHClient): if self.timeout: conninfo['timeout'] = self.timeout + extra = {'_hostname': self.hostname, '_port': self.port, + '_username': self.username, '_timeout': self.timeout} + self.logger.debug('Connecting to server', extra=extra) + self.client.connect(**conninfo) return True def put(self, path, contents=None, chmod=None, mode='w'): + extra = {'_path': path, '_mode': mode, '_chmod': chmod} + self.logger.debug('Uploading file', extra=extra) + sftp = self.client.open_sftp() # less than ideal, but we need to mkdir stuff otherwise file() fails head, tail = psplit(path) @@ -208,12 +230,18 @@ class ParamikoSSHClient(BaseSSHClient): return file_path def delete(self, path): + extra = {'_path': path} + self.logger.debug('Deleting file', extra=extra) + sftp = self.client.open_sftp() sftp.unlink(path) sftp.close() return True def run(self, cmd): + extra = {'_cmd': cmd} + self.logger.debug('Executing command', extra=extra) + # based on exec_command() bufsize = -1 t = self.client.get_transport() @@ -227,9 +255,15 @@ class ParamikoSSHClient(BaseSSHClient): status = chan.recv_exit_status() so = stdout.read() se = stderr.read() + + extra = {'_status': status, '_stdout': so, '_stderr': se} + self.logger.debug('Command finished', extra=extra) + return [so, se, status] def close(self): + self.logger.debug('Closing server connection') + self.client.close() return True @@ -288,17 +322,6 @@ class ShellOutSSHClient(BaseSSHClient): def close(self): return True - def _get_and_setup_logger(self): - logger = logging.getLogger('libcloud.compute.ssh') - path = os.getenv('LIBCLOUD_DEBUG') - - if path: - handler = logging.FileHandler(path) - logger.addHandler(handler) - logger.setLevel(logging.DEBUG) - - return logger - def _get_base_ssh_command(self): cmd = ['ssh'] http://git-wip-us.apache.org/repos/asf/libcloud/blob/df6dc611/libcloud/test/compute/test_ssh_client.py ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/test_ssh_client.py b/libcloud/test/compute/test_ssh_client.py index 840ff98..57df407 100644 --- a/libcloud/test/compute/test_ssh_client.py +++ b/libcloud/test/compute/test_ssh_client.py @@ -17,9 +17,12 @@ from __future__ import absolute_import from __future__ import with_statement +import os import sys +import tempfile import unittest +from libcloud import _init_once from libcloud.compute.ssh import ParamikoSSHClient from libcloud.compute.ssh import ShellOutSSHClient from libcloud.compute.ssh import have_paramiko @@ -41,6 +44,9 @@ class ParamikoSSHClientTests(unittest.TestCase): 'username': 'ubuntu', 'key': '~/.ssh/ubuntu_ssh', 'timeout': '600'} + _, self.tmp_file = tempfile.mkstemp() + os.environ['LIBCLOUD_DEBUG'] = self.tmp_file + _init_once() self.ssh_cli = ParamikoSSHClient(**conn_params) @patch('paramiko.SSHClient', Mock) @@ -64,6 +70,7 @@ class ParamikoSSHClientTests(unittest.TestCase): 'look_for_keys': False, 'port': 22} mock.client.connect.assert_called_once_with(**expected_conn) + self.assertLogMsg('Connecting to server') @patch('paramiko.SSHClient', Mock) def test_create_without_credentials(self): @@ -112,9 +119,12 @@ class ParamikoSSHClientTests(unittest.TestCase): mode='w') mock.run(sd) + # Make assertions over 'run' method mock_cli.get_transport().open_session().exec_command \ .assert_called_once_with(sd) + self.assertLogMsg('Executing command (cmd=/root/random_script.sh)') + self.assertLogMsg('Command finished') mock.close() @@ -131,8 +141,16 @@ class ParamikoSSHClientTests(unittest.TestCase): mock.delete(sd) # Make assertions over the 'delete' method mock.client.open_sftp().unlink.assert_called_with(sd) + self.assertLogMsg('Deleting file') mock.close() + self.assertLogMsg('Closing server connection') + + def assertLogMsg(self, expected_msg): + with open(self.tmp_file, 'r') as fp: + content = fp.read() + + self.assertTrue(content.find(expected_msg) != -1) if not ParamikoSSHClient: @@ -193,9 +211,9 @@ class ShellOutSSHClientTests(unittest.TestCase): self.assertEqual(cmd1, ['ssh', 'root@localhost']) self.assertEqual(cmd2, ['ssh', '-i', '/home/my.key', - 'root@localhost']) + 'root@localhost']) self.assertEqual(cmd3, ['ssh', '-i', '/home/my.key', - '-oConnectTimeout=5', 'root@localhost']) + '-oConnectTimeout=5', 'root@localhost']) if __name__ == '__main__': http://git-wip-us.apache.org/repos/asf/libcloud/blob/df6dc611/libcloud/utils/logging.py ---------------------------------------------------------------------- diff --git a/libcloud/utils/logging.py b/libcloud/utils/logging.py new file mode 100644 index 0000000..e95ca9c --- /dev/null +++ b/libcloud/utils/logging.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Taken from https://github.com/Kami/python-extra-log-formatters + +from __future__ import absolute_import + +import logging + +__all__ = [ + 'ExtraLogFormatter' +] + + +class ExtraLogFormatter(logging.Formatter): + """ + Custom log formatter which attaches all the attributes from the "extra" + dictionary which start with an underscore to the end of the log message. + + For example: + extra={'_id': 'user-1', '_path': '/foo/bar'} + """ + def format(self, record): + custom_attributes = dict([(k, v) for k, v in record.__dict__.items() + if k.startswith('_')]) + custom_attributes = self._dict_to_str(custom_attributes) + + msg = logging.Formatter.format(self, record) + msg = '%s (%s)' % (msg, custom_attributes) + return msg + + def _dict_to_str(self, dictionary): + result = ['%s=%s' % (k[1:], str(v)) for k, v in dictionary.items()] + result = ','.join(result) + return result
