Added --with-profiling, since I was tired of manually editing the
makefile. :) Adds -pg and suppresses LDFLAGS -s.
Ascii mode is extremely slow; this affects ls, too. This is because of
the data pipeline in ASCII mode: every line tends to cause a full
scheduling loop. Fixed this in a couple places:
FtpDirList.cc: moved EPLF handling to a separate function, so I could
make sense of Do().
Important: I don't know what EPLF is and have no means of testing that code
change. Do you have a server I could use to test it?
FtpDirList::Do: loop on input handling. The end result is the same
(except a single Do() call can give multiple lines).
FileCopyPeerFDStream::Do: loop on Get_LL and Put_LL until the buffer is
full (Get_LL) or no more data left (Put_LL) or no data is put or retrieved.
Of course, this still follows the regular nonblocking behavior.
This brings get -a up from 40k/sec to 300k/sec to localhost on my
p2/450; lftp no longer thrashes CPU (the bottleneck is proftpd.) "ls >
/dev/null" on a 10000 file dir out of cache is instantaneous.
Moved code to strip \r\n to misc.cc. Rewrote it to operate in linear
time; this makes nlist out of cache fast. There may be other places
this code can be used.
nlist status output is still broken. The only fix I can see is the one
I had originally, so I didn't bother repeating it. :)
--
Glenn Maynard
Index: configure.in
===================================================================
RCS file: /home/lav/cvsroot/lftp/configure.in,v
retrieving revision 1.110
diff -u -r1.110 configure.in
--- configure.in 2001/10/23 07:58:24 1.110
+++ configure.in 2001/10/26 18:32:50
@@ -25,6 +25,11 @@
[ with_debug=$withval; ],
[ with_debug=no; ])
+AC_ARG_WITH(profiling,
+[ --with-profiling enable profiling],
+[ with_profiling=$withval; ],
+[ with_profiling=no; ])
+
if test x$with_debug = xno; then
if test x$DEFAULT_CFLAGS = xyes; then
CFLAGS="`echo $CFLAGS | sed 's/-g//'`"
@@ -38,7 +43,8 @@
CXXFLAGS=-O
fi
fi
- if test x$DEFAULT_LDFLAGS = xyes; then
+ # don't strip when profiling
+ if test x$DEFAULT_LDFLAGS = xyes -a x$with_profiling != xyes; then
case "`uname -s`" in
Darwin) ;;
*) LDFLAGS="$LDFLAGS -s";;
@@ -56,6 +62,16 @@
fi
fi
+if test x$with_profiling = xyes; then
+ CFLAGS="$CFLAGS -pg"
+ CXXFLAGS="$CXXFLAGS -pg"
+ # profiling requires debugging, too, but don't nuke -O
+# if test x$with_debug != xyes; then
+# CFLAGS="$CFLAGS -g"
+# CXXFLAGS="$CXXFLAGS -g"
+# fi
+fi
+
if test x$GCC = xyes; then
CFLAGS="$CFLAGS -Wall"
fi
Index: src/FileCopy.cc
===================================================================
RCS file: /home/lav/cvsroot/lftp/src/FileCopy.cc,v
retrieving revision 1.72
diff -u -r1.72 FileCopy.cc
--- src/FileCopy.cc 2001/10/17 06:32:40 1.72
+++ src/FileCopy.cc 2001/10/26 18:32:50
@@ -861,28 +861,7 @@
Allocate(s);
memmove(buffer+buffer_ptr,b,s);
#ifndef NATIVE_CRLF
- if(ascii)
- {
- char *buf=buffer+buffer_ptr;
- // find where line ends.
- char *cr=buf;
- for(;;)
- {
- cr=(char *)memchr(cr,'\r',s-(cr-buf));
- if(!cr)
- break;
- if(cr-buf<s-1 && cr[1]=='\n')
- {
- memmove(cr,cr+1,s-(cr-buf)-1);
- s--;
- if(s<=1)
- break;
- }
- else if(cr-buf==s-1)
- break;
- cr++;
- }
- }
+ if(ascii) s = crlf_to_lf(buffer+buffer_ptr, s);
#endif // NATIVE_CRLF
in_buffer=s;
pos=seek_pos;
@@ -1273,7 +1252,6 @@
int FileCopyPeerFDStream::Do()
{
int m=STALL;
- int res;
if(Done() || Error())
return m;
switch(mode)
@@ -1299,31 +1277,39 @@
}
if(!write_allowed)
return m;
- res=Put_LL(buffer+buffer_ptr,in_buffer);
- if(res>0)
- {
- in_buffer-=res;
- buffer_ptr+=res;
- return MOVED;
+ while(in_buffer > 0) {
+ int res=Put_LL(buffer+buffer_ptr,in_buffer);
+ if(res>0)
+ {
+ in_buffer-=res;
+ buffer_ptr+=res;
+ m = MOVED;
+ }
+ if(res<0)
+ return MOVED;
+ if(res == 0)
+ break;
}
- if(res<0)
- return MOVED;
break;
case GET:
- if(eof)
- return m;
- res=Get_LL(GET_BUFSIZE);
- if(res>0)
- {
- in_buffer+=res;
- SaveMaxCheck(0);
- return MOVED;
+ while(in_buffer < GET_BUFSIZE) {
+ if(eof)
+ return MOVED;
+ int res=Get_LL(GET_BUFSIZE);
+ if(res>0)
+ {
+ in_buffer+=res;
+ SaveMaxCheck(0);
+ m = MOVED;
+ }
+ if(res<0)
+ return MOVED;
+ if(eof)
+ return MOVED;
+ if(res == 0)
+ break;
}
- if(res<0)
- return MOVED;
- if(eof)
- return MOVED;
break;
}
return m;
Index: src/FtpDirList.cc
===================================================================
RCS file: /home/lav/cvsroot/lftp/src/FtpDirList.cc,v
retrieving revision 1.15
diff -u -r1.15 FtpDirList.cc
--- src/FtpDirList.cc 2000/09/05 12:46:07 1.15
+++ src/FtpDirList.cc 2001/10/26 18:32:51
@@ -36,6 +36,99 @@
#define super DirList
+char *FtpDirList::EPLF(const char *b, int linelen)
+{
+ // check for EPLF listing
+ if(linelen <= 1) return NULL;
+ if(b[0]!='+') return NULL;
+
+ const char *scan=b+1;
+ int scan_len=linelen-1;
+
+ const char *name=0;
+ int name_len=0;
+ off_t size=NO_SIZE;
+ time_t date=NO_DATE;
+ bool dir=false;
+ int perms=-1;
+ while(scan && scan_len>0)
+ {
+ switch(*scan)
+ {
+ case '\t': // the rest is file name.
+ name=scan+1;
+ name_len=scan_len-1;
+ scan=0;
+ break;
+ case 's': {
+ long long size_ll;
+ if(1 != sscanf(scan+1,"%lld",&size_ll))
+ break;
+ size = size_ll;
+ break;
+ }
+ case 'm': {
+ long date_l;
+ if(1 != sscanf(scan+1,"%ld",&date_l))
+ break;
+ date = date_l;
+ break;
+ }
+ case '/':
+ dir=true;
+ break;
+ case 'r':
+ dir=false;
+ break;
+ case 'i':
+ break;
+ case 'u':
+ if(scan[1]=='p') // permissions.
+ sscanf(scan+2,"%o",&perms);
+ break;
+ default:
+ name=0;
+ scan=0;
+ break;
+ }
+ if(scan==0 || scan_len==0)
+ break;
+ const char *comma=find_char(scan,scan_len,',');
+ if(comma)
+ {
+ scan_len-=comma+1-scan;
+ scan=comma+1;
+ }
+ else
+ break;
+ }
+ if(!name || name_len == 0) return NULL;
+
+ // ok, this is EPLF. Format new string.
+ char *line_add=(char *) xmalloc(80+name_len);
+ if(perms==-1)
+ perms=(dir?0755:0644);
+ char size_str[32];
+ if(size==NO_SIZE)
+ strcpy(size_str,"-");
+ else
+ sprintf(size_str,"%lld",(long long)size);
+ char date_str[21];
+ if(date == NO_DATE)
+ strcpy(date_str,"-");
+ else {
+ struct tm *t=localtime(&date);
+ sprintf(date_str, "%04d-%02d-%02d %02d:%02d",
+ t->tm_year+1900,t->tm_mon+1,t->tm_mday,
+ t->tm_hour,t->tm_min);
+ }
+ sprintf(line_add, "%c%s %10s %16s %.*s",
+ dir ? 'd':'-', format_perms(perms), size_str,
+ date_str, name_len, name);
+
+ return line_add;
+}
+
int FtpDirList::Do()
{
if(done)
@@ -88,108 +181,31 @@
int m=STALL;
- if(len>0)
+ while(len>0)
{
const char *eol=find_char(b,len,'\n');
if(!eol && !ubuf->Eof() && len<0x1000)
return m;
- int skip_len=len;
+ int line_len=len;
+ char *eplf = NULL;
if(eol)
{
- skip_len=eol+1-b;
- len=skip_len;
+ line_len=eol+1-b;
+
+ eplf = EPLF(b, eol-b);
+ }
- const char *name=0;
- int name_len=0;
- off_t size=NO_SIZE;
- time_t date=NO_DATE;
- long date_l;
- long long size_ll;
- bool dir=false;
- int perms=-1;
- // check for EPLF listing
- if(eol>b+1 && b[0]=='+')
- {
- const char *scan=b+1;
- int scan_len=len-1;
- while(scan && scan_len>0)
- {
- switch(*scan)
- {
- case '\t': // the rest is file name.
- name=scan+1;
- name_len=scan_len-1;
- scan=0;
- break;
- case 's':
- if(1 != sscanf(scan+1,"%lld",&size_ll))
- break;
- size = size_ll;
- break;
- case 'm':
- if(1 != sscanf(scan+1,"%ld",&date_l))
- break;
- date = date_l;
- break;
- case '/':
- dir=true;
- break;
- case 'r':
- dir=false;
- break;
- case 'i':
- break;
- case 'u':
- if(scan[1]=='p') // permissions.
- sscanf(scan+2,"%o",&perms);
- break;
- default:
- name=0;
- scan=0;
- break;
- }
- if(scan==0 || scan_len==0)
- break;
- const char *comma=find_char(scan,scan_len,',');
- if(comma)
- {
- scan_len-=comma+1-scan;
- scan=comma+1;
- }
- else
- break;
- }
- if(name && name_len>0)
- {
- // ok, this is EPLF. Format new string.
- char *line_add=string_alloca(80+name_len);
- if(perms==-1)
- perms=(dir?0755:0644);
- char size_str[32];
- if(size==NO_SIZE)
- strcpy(size_str,"-");
- else
- sprintf(size_str,"%lld",(long long)size);
- char date_str[21];
- if(date == NO_DATE)
- strcpy(date_str,"-");
- else {
- struct tm *t=localtime(&date);
- sprintf(date_str, "%04d-%02d-%02d %02d:%02d",
- t->tm_year+1900,t->tm_mon+1,t->tm_mday,
- t->tm_hour,t->tm_min);
- }
- sprintf(line_add, "%c%s %10s %16s %.*s",
- dir ? 'd':'-', format_perms(perms), size_str,
- date_str, name_len, name);
- b=line_add;
- len=strlen(b);
- }
- } // end if(+).
- } // end if(eol)
+ if(eplf) {
+ /* put the new string instead */
+ buf->Put(eplf,strlen(eplf));
+ xfree(eplf);
+ } else
+ buf->Put(b,line_len);
+
+ ubuf->Skip(line_len);
+ len -= line_len;
+ b += line_len;
- buf->Put(b,len);
- ubuf->Skip(skip_len);
m=MOVED;
}
Index: src/FtpDirList.h
===================================================================
RCS file: /home/lav/cvsroot/lftp/src/FtpDirList.h,v
retrieving revision 1.4
diff -u -r1.4 FtpDirList.h
--- src/FtpDirList.h 1999/08/18 14:16:08 1.4
+++ src/FtpDirList.h 2001/10/26 18:32:51
@@ -29,6 +29,8 @@
Buffer *ubuf;
char *pattern;
+ static char *FtpDirList::EPLF(const char *line, int len);
+
public:
FtpDirList(ArgV *a,FileAccess *fa);
~FtpDirList();
Index: src/commands.cc
===================================================================
RCS file: /home/lav/cvsroot/lftp/src/commands.cc,v
retrieving revision 1.148
diff -u -r1.148 commands.cc
--- src/commands.cc 2001/10/25 13:40:28 1.148
+++ src/commands.cc 2001/10/26 18:32:52
@@ -1191,7 +1191,7 @@
{"long",no_argument,0,'l'},
{"quiet",no_argument,0,'q'},
{"size", no_argument,0,'s'}, /* show size */
- {"filesize", no_argument,0,0}, /* for files only */
+ {"filesize", no_argument,0,0}, /* for files only */
{"nocase", no_argument,0,'i'},
{"sortnocase", no_argument,0,'I'},
{"dirsfirst", no_argument,0,'D'},
Index: src/misc.cc
===================================================================
RCS file: /home/lav/cvsroot/lftp/src/misc.cc,v
retrieving revision 1.36
diff -u -r1.36 misc.cc
--- src/misc.cc 2001/10/25 13:42:57 1.36
+++ src/misc.cc 2001/10/26 18:32:53
@@ -708,3 +708,45 @@
va_end(va);
return ret;
}
+
+static char *mem_crlf(const char *buf, int s)
+{
+ while(buf && s) {
+ char *cr=(char *)memchr(buf,'\r',s);
+
+ if(!cr || cr == buf+s-1)
+ return NULL;
+
+ if(cr[1] == '\n') return cr;
+ s -= (cr-buf)+1;
+ buf += (cr-buf)+1;
+ }
+
+ return NULL;
+}
+
+/* convert \r\n -> \n in buf; return new size; be efficient for large strings */
+int crlf_to_lf(char *buf, int s)
+{
+ int retsiz = s;
+
+ char *startpos;
+ startpos = mem_crlf(buf, s);
+ if(!startpos) return retsiz;
+
+ char *append = startpos;
+ while(startpos+1 < buf + s)
+ {
+ char *nextpos = mem_crlf(startpos+1, s - (startpos+1 - buf));
+
+ if(nextpos == 0) nextpos = buf + s - 1; /* EOS */
+ memmove(append, startpos+1, nextpos-startpos);
+ append += nextpos-startpos-1;
+
+ startpos=nextpos;
+ retsiz--;
+ }
+
+ return retsiz;
+}
+
Index: src/misc.h
===================================================================
RCS file: /home/lav/cvsroot/lftp/src/misc.h,v
retrieving revision 1.25
diff -u -r1.25 misc.h
--- src/misc.h 2001/10/25 13:42:57 1.25
+++ src/misc.h 2001/10/26 18:32:53
@@ -109,4 +109,7 @@
char *xvasprintf(const char *format, va_list ap);
char *xasprintf(const char *format, ...);
+/* convert \r\n -> \n in buf; return new size */
+int crlf_to_lf(char *buf, int s);
+
#endif // MISC_H
