Revision: 167
Author: janne.t.harkonen
Date: Mon Aug 27 05:07:10 2012
Log: moved SFT functionality from library to client
http://code.google.com/p/robotframework-sshlibrary/source/detail?r=167
Modified:
/trunk/atest/get_file.txt
/trunk/atest/put_file.txt
/trunk/src/SSHLibrary/client.py
/trunk/src/SSHLibrary/javaclient.py
/trunk/src/SSHLibrary/library.py
/trunk/src/SSHLibrary/pythonclient.py
=======================================
--- /trunk/atest/get_file.txt Fri Aug 24 04:22:16 2012
+++ /trunk/atest/get_file.txt Mon Aug 27 05:07:10 2012
@@ -24,7 +24,7 @@
Get File And Verify Listing /home/${USERNAME}/* ${TMPDIR}${/}
${TEST SCRIPT NAME} ${INTERACTIVE TEST SCRIPT NAME} ${REPEAT TEST SCRIPT
NAME}
Getting Mulitple Source Files To Single File Fails
- Run Keyword And Expect Error ValueError: It is not possible to copy
multiple source files to one destination file. SSHLibrary.Get File
/home/${USERNAME}/*.sh ${TMPDIR}${/}foo
+ Run Keyword And Expect Error Cannot copy multiple source files to one
destination file. SSHLibrary.Get File /home/${USERNAME}/*.sh
${TMPDIR}${/}foo
Get File To Curdir
SSHLibrary.Get File /home/${USERNAME}/${TEST SCRIPT NAME} .
=======================================
--- /trunk/atest/put_file.txt Fri Aug 24 04:22:16 2012
+++ /trunk/atest/put_file.txt Mon Aug 27 05:07:10 2012
@@ -62,7 +62,7 @@
[Teardown] Execute Command rm -f ${TEST FILE NAME} ${TEST FILE 2
NAME}
Put File Should Fail When There Are No Source Files
- Run Keyword And Expect Error There were no source files
matching 'non*existing' SSHLibrary.Put File non*existing
+ Run Keyword And Expect Error There are no source files
matching 'non*existing' SSHLibrary.Put File non*existing
*** Keywords ***
Put Files And Verify
@@ -74,7 +74,7 @@
Put File And Specify Newlines
[Arguments] ${source} ${destination} ${newlines} @{expected}
Vefify Remote Files do Not Exist @{expected}
- Put File ${source} ${destination} newlines=${newlines}
+ Put File ${source} ${destination} newline=${newlines}
Verify Remote Files Exist ${EMPTY} @{expected}
Vefify Remote Files do Not Exist
=======================================
--- /trunk/src/SSHLibrary/client.py Mon Aug 27 05:06:42 2012
+++ /trunk/src/SSHLibrary/client.py Mon Aug 27 05:07:10 2012
@@ -12,10 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import with_statement
+
import sys
import os
import re
import time
+import glob
+
+from robot import utils
from .core import SSHClientException, TimeEntry
@@ -234,17 +239,154 @@
if not self.config.prompt:
raise SSHClientException('Prompt is not set.')
- def put_file(self, source, dest, mode, newline_char):
+ def put_file(self, source, destination='.', mode='0744',
+ newline='default', path_separator='/'):
+ """Put file(s) from localhost to remote host.
+
+ :param source: Local file path. May be a simple pattern containing
+ '*' and '?', in which case all matching files are tranferred
+ :param destintation: Remote path. If many files are transferred,
+ must be a directory. Defaults to users home directory.
+ :param mode: File permissions for the remote file. Defined as a
+ Unix file format string, e.g. '0600'
+ :param newline: Newline character to be used in the remote file.
+ Default is 'LF', i.e. the line feed character.
+ :param path_separator: The path separator on the remote machine.
+ Must be defined if the remote machine runs Windows.
+ """
+ sftp_client = self._create_sftp_client()
+ sources, destinations = sftp_client.put_file(
+ source, destination, mode, newline, path_separator)
+ sftp_client.close()
+ return sources, destinations
+
+ def get_file(self, source, destination='.', path_separator='/'):
+ """Get file(s) from the remote host to localhost.
+
+ :param source: Remote file path. May be a simple pattern containing
+ '*' and '?', in which case all matching files are tranferred
+ :param destintation: Local path. If many files are transferred,
+ must be a directory. Defaults to current working directory.
+ :param path_separator: The path separator on the remote machine.
+ Must be defined if the remote machine runs Windows.
+ """
+ sftp_client = self._create_sftp_client()
+ sources, destinations = sftp_client.get_file(
+ source, destination, path_separator)
+ sftp_client.close()
+ return sources, destinations
+
+
+class AbstractSFTPClient(object):
+
+ def __init__(self, ssh_client):
+ self._client = self._create_client(ssh_client)
+ self._homedir = self._resolve_homedir()
+
+ def close(self):
+ self._client.close()
+
+ def get_file(self, sources, destination, path_separator='/'):
+ remotefiles = self._get_get_file_sources(sources, path_separator)
+ localfiles = self._get_get_file_destinations(remotefiles,
destination)
+ for src, dst in zip(remotefiles, localfiles):
+ self._get_file(src, dst)
+ return remotefiles, localfiles
+
+ def _get_get_file_sources(self, source, path_separator):
+ if path_separator in source:
+ path, pattern = source.rsplit(path_separator, 1)
+ else:
+ path, pattern = '', source
+ if not path:
+ path = '.'
+ sourcefiles = []
+ for filename in self._listfiles(path):
+ if utils.matches(filename, pattern):
+ if path:
+ filename = path_separator.join([path, filename])
+ sourcefiles.append(filename)
+ if not sourcefiles:
+ msg = "There were no source files matching '%s'" % source
+ raise SSHClientException(msg)
+ return sourcefiles
+
+ def _get_get_file_destinations(self, sourcefiles, dest):
+ if dest == '.':
+ dest += os.sep
+ is_dir = dest.endswith(os.sep)
+ if not is_dir and len(sourcefiles) > 1:
+ msg = 'Cannot copy multiple source files to one destination
file.'
+ raise SSHClientException(msg)
+ dest = os.path.abspath(dest.replace('/', os.sep))
+ self._create_missing_local_dirs(dest, is_dir)
+ if is_dir:
+ return [os.path.join(dest, os.path.split(name)[1])
+ for name in sourcefiles]
+ return [dest]
+
+ def _create_missing_local_dirs(self, dest, is_dir):
+ if not is_dir:
+ dest = os.path.dirname(dest)
+ if not os.path.exists(dest):
+ os.makedirs(dest)
+
+ def put_file(self, sources, destination, mode, newline,
+ path_separator='/'):
+ mode = int(mode, 8)
+ newline = {'CRLF': '\r\n', 'LF': '\n'}.get(newline.upper(), None)
+ localfiles = self._get_put_file_sources(sources)
+ remotefiles, remotedir = self._get_put_file_destinations(
+ localfiles, destination, path_separator)
+ self._create_missing_remote_path(remotedir)
+ for src, dst in zip(localfiles, remotefiles):
+ self._put_file(src, dst, mode, newline)
+ return localfiles, remotefiles
+
+ def _put_file(self, source, dest, mode, newline_char):
remotefile = self._create_remote_file(dest, mode)
- localfile = open(source, 'rb')
- position = 0
- while True:
- data = localfile.read(4096)
- if not data:
- break
- if newline_char and '\n' in data:
- data = data.replace('\n', newline_char)
- self._write_to_remote_file(remotefile, data, position)
- position += len(data)
- self._close_remote_file(remotefile)
- localfile.close()
+ with open(source, 'rb') as localfile:
+ position = 0
+ while True:
+ data = localfile.read(4096)
+ if not data:
+ break
+ if newline_char and '\n' in data:
+ data = data.replace('\n', newline_char)
+ self._write_to_remote_file(remotefile, data, position)
+ position += len(data)
+ self._close_remote_file(remotefile)
+
+ def _parse_path_elements(self, dest, path_separator):
+ def _isabs(path):
+ if dest.startswith(path_separator):
+ return True
+ if path_separator == '\\' and path[1:3] == ':\\':
+ return True
+ return False
+ if not _isabs(dest):
+ dest = path_separator.join([self._homedir, dest])
+ return dest.rsplit(path_separator, 1)
+
+ def _get_put_file_sources(self, source):
+ sources = [f for f in glob.glob(source.replace('/', os.sep))
+ if os.path.isfile(f)]
+ if not sources:
+ msg = "There are no source files matching '%s'" % source
+ raise SSHClientException(msg)
+ return sources
+
+ def _get_put_file_destinations(self, sources, dest, path_separator):
+ dest = dest.split(':')[-1].replace('\\', '/')
+ if dest == '.':
+ dest = self._homedir + '/'
+ if len(sources) > 1 and dest[-1] != '/':
+ raise ValueError('It is not possible to copy multiple source '
+ 'files to one destination file.')
+ dirpath, filename = self._parse_path_elements(dest, path_separator)
+ if filename:
+ files = [path_separator.join([dirpath, filename])]
+ else:
+ files = [path_separator.join([dirpath, os.path.split(path)[1]])
+ for path in sources]
+ return files, dirpath
=======================================
--- /trunk/src/SSHLibrary/javaclient.py Sun Aug 26 21:30:21 2012
+++ /trunk/src/SSHLibrary/javaclient.py Mon Aug 27 05:07:10 2012
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
import jarray
from java.io import (File, BufferedReader, InputStreamReader, IOException,
FileOutputStream)
@@ -23,7 +22,7 @@
raise ImportError('Importing Trilead SSH classes failed. '
'Make sure you have the Trilead jar file in
CLASSPATH.')
-from .client import AbstractSSHClient
+from .client import AbstractSSHClient, AbstractSFTPClient
from .core import SSHClientException, Command
@@ -87,57 +86,61 @@
return chr(buf[0])
return ''
- def create_sftp_client(self):
- self.sftp_client = SFTPv3Client(self.client)
- self.homedir = self.sftp_client.canonicalPath('.') + '/'
+ def _create_sftp_client(self):
+ return SFTPClient(self.client)
- def close_sftp_client(self):
- self.sftp_client.close()
- def create_missing_remote_path(self, path):
+class SFTPClient(AbstractSFTPClient):
+
+ def _create_client(self, ssh_client):
+ return SFTPv3Client(ssh_client)
+
+ def _resolve_homedir(self):
+ return self._client.canonicalPath('.') + '/'
+
+ def _create_missing_remote_path(self, path):
if path.startswith('/'):
curdir = '/'
else:
- curdir = self.sftp_client.canonicalPath('.')
+ curdir = self._client.canonicalPath('.')
for dirname in path.split('/'):
if dirname:
curdir = '%s/%s' % (curdir, dirname)
try:
- self.sftp_client.stat(curdir)
+ self._client.stat(curdir)
except IOException:
- print "*INFO* Creating missing remote directory '%s'" %
curdir
- self.sftp_client.mkdir(curdir, 0744)
+ self._client.mkdir(curdir, 0744)
def _create_remote_file(self, dest, mode):
- remotefile = self.sftp_client.createFile(dest)
+ remotefile = self._client.createFile(dest)
try:
- tempstats = self.sftp_client.fstat(remotefile)
+ tempstats = self._client.fstat(remotefile)
tempstats.permissions = mode
- self.sftp_client.fsetstat(remotefile, tempstats)
+ self._client.fsetstat(remotefile, tempstats)
except SFTPException:
pass
return remotefile
def _write_to_remote_file(self, remotefile, data, position):
- self.sftp_client.write(remotefile, position, data, 0, len(data))
+ self._client.write(remotefile, position, data, 0, len(data))
def _close_remote_file(self, remotefile):
- self.sftp_client.closeFile(remotefile)
+ self._client.closeFile(remotefile)
- def listfiles(self, path):
- return [finfo.filename for finfo in self.sftp_client.ls(path) if
+ def _listfiles(self, path):
+ return [finfo.filename for finfo in self._client.ls(path) if
finfo.attributes.getOctalPermissions().startswith('0100')]
- def get_file(self, source, dest):
+ def _get_file(self, source, dest):
localfile = FileOutputStream(dest)
- tempstats = self.sftp_client.stat(source)
+ tempstats = self._client.stat(source)
remotefilesize = tempstats.size
- remotefile = self.sftp_client.openFileRO(source)
+ remotefile = self._client.openFileRO(source)
size = 0
arraysize = 4096
data = jarray.zeros(arraysize, 'b')
while True:
- moredata = self.sftp_client.read(remotefile, size, data, 0,
+ moredata = self._client.read(remotefile, size, data, 0,
arraysize)
datalen = len(data)
if moredata == -1:
@@ -146,7 +149,7 @@
datalen = remotefilesize - size
localfile.write(data, 0, datalen)
size += datalen
- self.sftp_client.closeFile(remotefile)
+ self._client.closeFile(remotefile)
localfile.flush()
localfile.close()
=======================================
--- /trunk/src/SSHLibrary/library.py Mon Aug 27 05:06:42 2012
+++ /trunk/src/SSHLibrary/library.py Mon Aug 27 05:07:10 2012
@@ -1,9 +1,3 @@
-import os
-import glob
-import posixpath
-
-from robot import utils
-
from .client import SSHClient, AbstractSSHClient
from .connectioncache import ConnectionCache
from .core import DefaultConfig, ClientConfig, SSHClientException
@@ -449,7 +443,39 @@
self._log(output, loglevel)
return output
- def put_file(self, source, destination='.', mode='0744',
newlines='default'):
+ def get_file(self, source, destination='.'):
+ """Copies file(s) from remote host to local host.
+
+ 1. If the destination is an existing file, the source file is
copied
+ over it.
+ 2. If the destination is an existing directory, the source file is
+ copied into it. Possible file with same name is overwritten.
+ 3. If the destination does not exist and it ends with path
separator
+ ('/' in unixes, '\\' in Windows), it is considered a directory.
+ That directory is created and source file copied into it.
Possible
+ missing intermediate directories are also created.
+ 4. If the destination does not exist and it does not end with path
+ separator, it is considered a file. If the path to the file does
+ not exist it is created.
+ 5. If the destination is not given, the current working directory
in
+ the local machine is used as destination. This will most
probably
+ be the directory where test execution was started.
+
+ Using wild cards like '*' and '?' are allowed in the source.
+ When wild cards are used, destination MUST be a directory, and
files
+ matching the pattern are copied, but sub directories are ignored.
If
+ the contents of sub directories are also needed, use the keyword
again.
+
+ Examples:
+ | Get File | /path_to_remote_file/remote_file.txt |
/path_to_local_file/local_file.txt | # single file |
+ | Get File | /path_to_remote_files/*.txt |
/path_to_local_files/ | # multiple files with wild cards |
+
+ """
+ return self._run_sftp_command(self.ssh_client.get_file, source,
+ destination)
+
+ def put_file(self, source, destination='.', mode='0744',
+ newline='default'):
"""Copies file(s) from local host to remote host.
1. If the destination is an existing file, the source file is
copied
@@ -466,7 +492,7 @@
5. If destination is not given, the user's home directory
in the remote machine is used as destination.
- Using wild cards like '*' and '?' are allowed.
+ Using wild cards like '*' and '?' are allowed in the source.
When wild cards are used, destination MUST be a directory and only
files are copied from the source, sub directories are ignored. If
the
contents of sub directories are also needed, use the keyword again.
@@ -474,133 +500,25 @@
Default file permission is 0744 (-rwxr--r--) and can be changed by
giving a value to the optional `mode` parameter.
- `newlines` can be used to force newline characters that are
written to
+ `newline` can be used to force newline characters that are written
to
the remote file. Valid values are `CRLF` (for Windows) and `LF`.
Examples:
-
| Put File | /path_to_local_file/local_file.txt |
/path_to_remote_file/remote_file.txt | # single file
| |
| Put File | /path_to_local_files/*.txt |
/path_to_remote_files/ | # multiple files with wild cards
| |
| Put File | /path_to_local_files/*.txt |
/path_to_remote_files/ | 0777 | CRLF | # file permissions and forcing
Windows newlines |
"""
- mode = int(mode, 8)
- self.ssh_client.create_sftp_client()
- localfiles = self._get_put_file_sources(source)
- remotefiles, remotepath =
self._get_put_file_destinations(localfiles,
-
destination)
- self.ssh_client.create_missing_remote_path(remotepath)
- for src, dst in zip(localfiles, remotefiles):
- self._info("Putting '%s' to '%s'" % (src, dst))
- newline = {'CRLF': '\r\n', 'LF': '\n'}.get(newlines, None)
- self.ssh_client.put_file(src, dst, mode, newline)
- self.ssh_client.close_sftp_client()
+ cmd = self.ssh_client.put_file
+ return self._run_sftp_command(cmd, source, destination, mode,
newline)
- def _get_put_file_sources(self, source):
- sources = [f for f in glob.glob(source.replace('/', os.sep))
- if os.path.isfile(f)]
- if not sources:
- raise AssertionError("There were no source files
matching '%s'" %
- source)
- self._debug('Source pattern matched local files: %s' %
- utils.seq2str(sources))
- return sources
-
- def _get_put_file_destinations(self, sources, dest):
- dest = dest.split(':')[-1].replace('\\', '/')
- if dest == '.':
- dest = self.ssh_client.homedir + '/'
- if len(sources) > 1 and dest[-1] != '/':
- raise ValueError('It is not possible to copy multiple source '
- 'files to one destination file.')
- dirpath, filename = self._parse_path_elements(dest)
- if filename:
- files = [posixpath.join(dirpath, filename)]
- else:
- files = [posixpath.join(dirpath, os.path.split(path)[1])
- for path in sources]
- return files, dirpath
-
- def _parse_path_elements(self, dest):
- if not posixpath.isabs(dest):
- dest = posixpath.join(self.ssh_client.homedir, dest)
- return posixpath.split(dest)
-
- def get_file(self, source, destination='.'):
- """Copies a file from remote host to local host.
-
- 1. If the destination is an existing file, the source file is
copied
- over it.
- 2. If the destination is an existing directory, the source file is
- copied into it. Possible file with same name is overwritten.
- 3. If the destination does not exist and it ends with path
separator
- ('/' in unixes, '\\' in Windows), it is considered a directory.
- That directory is created and source file copied into it.
Possible
- missing intermediate directories are also created.
- 4. If the destination does not exist and it does not end with path
- separator, it is considered a file. If the path to the file does
- not exist it is created.
- 5. If the destination is not given, the current working directory
in
- the local machine is used as destination. This will most
probably
- be the directory where test execution was started.
-
- Using wild cards like '*' and '?' are allowed.
- When wild cards are used, destination MUST be a directory, and
files
- matching the pattern are copied, but sub directories are ignored.
If
- the contents of sub directories are also needed, use the keyword
again.
-
- Examples:
-
- | Get File | /path_to_remote_file/remote_file.txt |
/path_to_local_file/local_file.txt | # single file |
- | Get File | /path_to_remote_files/*.txt |
/path_to_local_files/ | # multiple files with wild cards |
-
- """
- self.ssh_client.create_sftp_client()
- remotefiles = self._get_get_file_sources(source)
- self._debug('Source pattern matched remote files: %s' %
- utils.seq2str(remotefiles))
- localfiles = self._get_get_file_destinations(remotefiles,
destination)
- for src, dst in zip(remotefiles, localfiles):
- self._info('Getting %s to %s' % (src, dst))
- self.ssh_client.get_file(src, dst)
- self.ssh_client.close_sftp_client()
-
- def _get_get_file_sources(self, source):
- path, pattern = posixpath.split(source)
- if not path:
- path = '.'
- sourcefiles = []
- for filename in self.ssh_client.listfiles(path):
- if utils.matches(filename, pattern):
- if path:
- filename = posixpath.join(path, filename)
- sourcefiles.append(filename)
- if not sourcefiles:
- raise AssertionError("There were no source files
matching '%s'" %
- source)
- return sourcefiles
-
- def _get_get_file_destinations(self, sourcefiles, dest):
- if dest == '.':
- dest += os.sep
- is_dir = dest.endswith(os.sep)
- if not is_dir and len(sourcefiles) > 1:
- raise ValueError('It is not possible to copy multiple source '
- 'files to one destination file.')
- dest = os.path.abspath(dest.replace('/', os.sep))
- self._create_missing_local_dirs(dest, is_dir)
- if is_dir:
- return [os.path.join(dest, os.path.split(name)[1])
- for name in sourcefiles]
- return [dest]
-
- def _create_missing_local_dirs(self, dest, is_dir):
- if not is_dir:
- dest = os.path.dirname(dest)
- if not os.path.exists(dest):
- self._info("Creating missing local directories for path '%s'" %
- dest)
- os.makedirs(dest)
+ def _run_sftp_command(self, command, *args):
+ try:
+ sources, destinations = command(*args)
+ except SSHClientException, e:
+ raise RuntimeError(e)
+ for src, dst in zip(sources, destinations):
+ self._info("'%s' -> '%s'" % (src, dst))
def _info(self, msg):
self._log(msg, 'INFO')
=======================================
--- /trunk/src/SSHLibrary/pythonclient.py Sun Aug 26 21:30:21 2012
+++ /trunk/src/SSHLibrary/pythonclient.py Mon Aug 27 05:07:10 2012
@@ -22,7 +22,7 @@
'Ensure that paramiko and pycrypto modules are installed.'
)
-from .client import AbstractSSHClient
+from .client import AbstractSSHClient, AbstractSFTPClient
from .core import Command, SSHClientException
@@ -85,47 +85,52 @@
return self.shell.recv(1)
return ''
- def create_sftp_client(self):
- self.sftp_client = self.client.open_sftp()
- self.homedir = self.sftp_client.normalize('.') + '/'
+ def _create_sftp_client(self):
+ return SFTPClient(self.client)
- def close_sftp_client(self):
- self.sftp_client.close()
- def create_missing_remote_path(self, path):
+class SFTPClient(AbstractSFTPClient):
+
+ def _create_client(self, ssh_client):
+ return ssh_client.open_sftp()
+
+ def _resolve_homedir(self):
+ return self._client.normalize('.') + '/'
+
+ def _get_file(self, source, dest):
+ self._client.get(source, dest)
+
+ def _write_to_remote_file(self, remotefile, data, position):
+ remotefile.write(data)
+
+ def _close_remote_file(self, remotefile):
+ remotefile.close()
+
+ def _create_missing_remote_path(self, path):
if path == '.':
return
if posixpath.isabs(path):
- self.sftp_client.chdir('/')
+ self._client.chdir('/')
else:
- self.sftp_client.chdir('.')
+ self._client.chdir('.')
for dirname in path.split('/'):
- if dirname and dirname not in
self.sftp_client.listdir(self.sftp_client.getcwd()):
- print "*INFO* Creating missing remote directory '%s'" %
dirname
- self.sftp_client.mkdir(dirname)
- self.sftp_client.chdir(dirname)
+ cwd = self._client.getcwd()
+ if dirname and dirname not in self._client.listdir(cwd):
+ self._client.mkdir(dirname)
+ self._client.chdir(dirname)
def _create_remote_file(self, dest, mode):
- remotfile = self.sftp_client.file(dest, 'wb')
+ remotfile = self._client.file(dest, 'wb')
remotfile.set_pipelined(True)
- self.sftp_client.chmod(dest, mode)
+ self._client.chmod(dest, mode)
return remotfile
- def _write_to_remote_file(self, remotefile, data, position):
- remotefile.write(data)
-
- def _close_remote_file(self, remotefile):
- remotefile.close()
-
- def listfiles(self, path):
+ def _listfiles(self, path):
return [getattr(fileinfo, 'filename', '?') for fileinfo
- in self.sftp_client.listdir_attr(path)
+ in self._client.listdir_attr(path)
if stat.S_ISREG(fileinfo.st_mode) or
stat.S_IFMT(fileinfo.st_mode) == 0]
- def get_file(self, source, dest):
- self.sftp_client.get(source, dest)
-
class RemoteCommand(Command):
