It seems all is not quite right with our old friend sftp_read.
It is very unhappy reading from files with a buffer bigger than the
file. In these cases is actually *sends* more than it receives. When
reading a sequence of such files, the reads get successively slower
after each file is opened, read and closed. Using the very scientific
method of stopping the debugger many times at random, it seems to be
spending most of its time in sftp_packetlist_flush.
My guess is that we are sending vastly more read-ahead requests than
we should. The number we send is based on the buffer size but capped
to LIBSSH2_CHANNEL_WINDOW_DEFAULT*4 (1MB). What I don't understand is
why this causes more of a problem the more files we try to read. If
it's a problem, fine, but surely it's a constant problem?
I've attached the script I'm using to reproduce the results. Execute
it as `libssh2_bigread ip_address username password
/path/to/small/file`. The script opens, reads and closes that small
file repeatedly, forever.
The trace output [1] shows an interesting time gap before each
"Closing handle" line which gets bigger and bigger with each iteration
during which nothing appears in the trace. Could it be waiting on the
socket in BLOCK_ADJUST here? The local CPU is at about 80% the entire
time.
Thoughts, suspicions and suggestions most welcome.
Alex
[1] http://dl.dropbox.com/u/6028779/read2.zip
--
Swish - Easy SFTP for Windows Explorer (http://www.swish-sftp.org)
/*
* The sample code has default values for host name, user name, password
* and path to copy, but you can specify them on the command line like:
*
* "sftp 192.168.0.1 user password /tmp/secrets -p|-i|-k"
*/
//#define TEST_READ_SIZE 6543210
#define TEST_READ_SIZE 654
#include <libssh2_config.h>
#include <libssh2.h>
#include <libssh2_sftp.h>
#ifdef HAVE_WINSOCK2_H
# include <winsock2.h>
#endif
#ifdef HAVE_NETINET_IN_H
# include <netinet/in.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
#endif
# ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_ARPA_INET_H
# include <arpa/inet.h>
#endif
#ifdef HAVE_SYS_TIME_H
# include <sys/time.h>
#endif
#include <sys/types.h>
#include <fcntl.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
const char *username="username";
const char *password="password";
const char *sftppath="/tmp/TEST";
static void kbd_callback(const char *name, int name_len,
const char *instruction, int instruction_len, int num_prompts,
const LIBSSH2_USERAUTH_KBDINT_PROMPT *prompts,
LIBSSH2_USERAUTH_KBDINT_RESPONSE *responses,
void **abstract)
{
(void)name;
(void)name_len;
(void)instruction;
(void)instruction_len;
if (num_prompts == 1) {
responses[0].text = strdup(password);
responses[0].length = strlen(password);
}
(void)prompts;
(void)abstract;
} /* kbd_callback */
int main(int argc, char *argv[])
{
unsigned long hostaddr;
int sock, i, auth_pw = 0;
struct sockaddr_in sin;
const char *fingerprint;
char *userauthlist;
LIBSSH2_SESSION *session;
int rc;
LIBSSH2_SFTP *sftp_session;
LIBSSH2_SFTP_HANDLE *sftp_handle;
char *mem;
#ifdef WIN32
WSADATA wsadata;
WSAStartup(MAKEWORD(2,0), &wsadata);
#endif
if (argc > 1) {
hostaddr = inet_addr(argv[1]);
} else {
hostaddr = htonl(0x7F000001);
}
if(argc > 2) {
username = argv[2];
}
if(argc > 3) {
password = argv[3];
}
if(argc > 4) {
sftppath = argv[4];
}
/*
* The application code is responsible for creating the socket
* and establishing the connection
*/
sock = socket(AF_INET, SOCK_STREAM, 0);
sin.sin_family = AF_INET;
sin.sin_port = htons(22);
sin.sin_addr.s_addr = hostaddr;
if (connect(sock, (struct sockaddr*)(&sin),
sizeof(struct sockaddr_in)) != 0) {
fprintf(stderr, "failed to connect!\n");
return -1;
}
/* Create a session instance
*/
session = libssh2_session_init();
if(!session)
return -1;
/* Since we have set non-blocking, tell libssh2 we are blocking */
libssh2_session_set_blocking(session, 1);
/* ... start it up. This will trade welcome banners, exchange keys,
* and setup crypto, compression, and MAC layers
*/
rc = libssh2_session_startup(session, sock);
if(rc) {
fprintf(stderr, "Failure establishing SSH session: %d\n", rc);
return -1;
}
/* At this point we havn't yet authenticated. The first thing to do
* is check the hostkey's fingerprint against our known hosts Your app
* may have it hard coded, may go to a file, may present it to the
* user, that's your call
*/
fingerprint = libssh2_hostkey_hash(session, LIBSSH2_HOSTKEY_HASH_MD5);
fprintf(stderr, "Fingerprint: ");
for(i = 0; i < 16; i++) {
fprintf(stderr, "%02X ", (unsigned char)fingerprint[i]);
}
fprintf(stderr, "\n");
/* check what authentication methods are available */
userauthlist = libssh2_userauth_list(session, username, strlen(username));
printf("Authentication methods: %s\n", userauthlist);
if (strstr(userauthlist, "password") != NULL) {
auth_pw |= 1;
}
if (strstr(userauthlist, "keyboard-interactive") != NULL) {
auth_pw |= 2;
}
/* if we got an 4. argument we set this option if supported */
if(argc > 5) {
if ((auth_pw & 1) && !strcasecmp(argv[5], "-p")) {
auth_pw = 1;
}
if ((auth_pw & 2) && !strcasecmp(argv[5], "-i")) {
auth_pw = 2;
}
}
if (auth_pw & 1) {
/* We could authenticate via password */
if (libssh2_userauth_password(session, username, password)) {
fprintf(stderr, "Authentication by password failed.\n");
goto shutdown;
}
} else if (auth_pw & 2) {
/* Or via keyboard-interactive */
if (libssh2_userauth_keyboard_interactive(session, username, &kbd_callback) ) {
printf("\tAuthentication by keyboard-interactive failed!\n");
goto shutdown;
} else {
printf("\tAuthentication by keyboard-interactive succeeded.\n");
}
} else {
printf("No supported authentication methods found!\n");
goto shutdown;
}
fprintf(stderr, "libssh2_sftp_init()!\n");
sftp_session = libssh2_sftp_init(session);
if (!sftp_session) {
fprintf(stderr, "Unable to init SFTP session\n");
goto shutdown;
}
libssh2_trace(session, (~0));
//libssh2_trace(session, (~0) & ~(LIBSSH2_TRACE_TRANS | LIBSSH2_TRACE_CONN));
//libssh2_trace(session, LIBSSH2_TRACE_CONN);
//for (int i = 6970001; i < 60000000; i += 1)
//for (int i = 100000; i < 100000000; i++)
i = 500000;
while (true)
{
//fprintf(stderr, "libssh2_sftp_open()!\n");
/* Request a file via SFTP */
sftp_handle =
libssh2_sftp_open(sftp_session, sftppath, LIBSSH2_FXF_READ, 0);
if (!sftp_handle) {
fprintf(
stderr, "Unable to open file with SFTP: %d\n",
libssh2_sftp_last_error(sftp_session));
goto shutdown;
}
//fprintf(stderr, "libssh2_sftp_open() is done, now receive data!\n");
//libssh2_sftp_seek(sftp_handle, 0);
mem = (char *)malloc(i);
fprintf(stderr, "trying buffer size %d\n", i);
int bytes_read = 0;
do
{
rc = libssh2_sftp_read(sftp_handle, mem, i - bytes_read);
if (rc < 0) {
char *msg;
fprintf(stderr, "READ FAILED!\n");
libssh2_session_last_error(session, &msg, NULL, 0);
fprintf(stderr, "%s\n", msg);
fprintf(stderr, "%s\n", libssh2_version(0));
break;
}
bytes_read += rc;
}
while (rc != 0 && bytes_read < i);
fprintf(stderr, "closing SFTP handle\n");
rc = libssh2_sftp_close_handle(sftp_handle);
fprintf(stderr, "libssh2_sftp_close_handle returned %d\n", rc);
free(mem);
if (bytes_read < i)
{
/*fprintf(stderr, "READ TOO OPTIMISTIC!\n");
fprintf(
stderr,
"it thinks it's finished but only read %d bytes\n", bytes_read);
fprintf(stderr, "%s\n", libssh2_version(0));
*/
//break;
}
else if (bytes_read == i)
{
fprintf(stderr, "READ SUCCEEDED!\n");
}
else if (bytes_read > i)
{
fprintf(stderr, "WTF?! BUFFER OVERFLOW!!\n");
fprintf(
stderr, "claims to have read %d bytes\n", bytes_read);
fprintf(stderr, "%s\n", libssh2_version(0));
break;
}
}
libssh2_sftp_shutdown(sftp_session);
shutdown:
libssh2_session_disconnect(session, "Normal Shutdown, Thank you for playing");
libssh2_session_free(session);
#ifdef WIN32
closesocket(sock);
#else
close(sock);
#endif
fprintf(stderr, "all done\n");
return 0;
}
_______________________________________________
libssh2-devel http://cool.haxx.se/cgi-bin/mailman/listinfo/libssh2-devel
