Downloading things can go a lot faster if the server and client support
http compression. This is easily added to the ftp program's http support.
It consists of two parts. Support for deflating the data we receive and
support for the chunked transfer the server will use to send data to us.
The chunked supported is probably useful in its own right as well.
Everything is ifndef small, of course.
Index: Makefile
===================================================================
RCS file: /home/tedu/cvs/src/usr.bin/ftp/Makefile,v
retrieving revision 1.25
diff -u -r1.25 Makefile
--- Makefile 5 May 2009 19:35:30 -0000 1.25
+++ Makefile 9 Jan 2011 21:14:51 -0000
@@ -17,8 +17,8 @@
CPPFLAGS+= -DINET6
-LDADD+= -ledit -lcurses -lutil -lssl -lcrypto
-DPADD+= ${LIBEDIT} ${LIBCURSES} ${LIBUTIL}
+LDADD+= -ledit -lcurses -lutil -lssl -lcrypto -lz
+DPADD+= ${LIBEDIT} ${LIBCURSES} ${LIBUTIL} ${LIBZ}
LDSTATIC= ${STATIC}
#COPTS+= -Wall -Wconversion -Wstrict-prototypes -Wmissing-prototypes
Index: fetch.c
===================================================================
RCS file: /home/tedu/cvs/src/usr.bin/ftp/fetch.c,v
retrieving revision 1.103
diff -u -r1.103 fetch.c
--- fetch.c 25 Aug 2010 20:32:37 -0000 1.103
+++ fetch.c 9 Jan 2011 21:15:42 -0000
@@ -63,6 +63,7 @@
#ifndef SMALL
#include <openssl/ssl.h>
#include <openssl/err.h>
+#include <zlib.h>
#else /* !SMALL */
#define SSL void
#endif /* !SMALL */
@@ -167,6 +168,74 @@
return (epath);
}
+#ifndef SMALL
+static size_t
+chunked_read(FILE *fin, SSL *ssl, unsigned char *buf, size_t amt, int gzipped)
+{
+ static int chunksize;
+ static int zinit;
+ static z_stream zctx;
+ static unsigned char zbuf[4096];
+
+ size_t chunkbuflen;
+ char *chunkbuf;
+ size_t len, zlen, zamt;
+ int rv;
+
+ if (!chunksize) {
+ chunkbuf = ftp_readline(fin, ssl, &chunkbuflen);
+ if (!chunkbuf) {
+ warnx("no chunk size");
+ return 0;
+ }
+ chunksize = strtol(chunkbuf, NULL, 16);
+ }
+ if (gzipped) {
+ if (zinit == -1) {
+ zinit = 0;
+ return 0;
+ }
+ if (zctx.avail_in == 0) {
+ zamt = sizeof(zbuf);
+ if (zamt > chunksize)
+ zamt = chunksize;
+ zlen = ftp_read(fin, ssl, zbuf, zamt);
+ chunksize -= zlen;
+ if (chunksize == 0) /* eat empty line after chunk */
+ ftp_readline(fin, ssl, &chunkbuflen);
+ zctx.avail_in = zlen;
+ zctx.next_in = zbuf;
+ }
+ if (!zinit) {
+ rv = inflateInit2(&zctx, 15 + 32);
+ zinit = 1;
+ }
+ zctx.next_out = buf;
+ zctx.avail_out = amt;
+ if ((rv = inflate(&zctx, Z_NO_FLUSH)) != Z_OK) {
+ if (rv == Z_STREAM_END) {
+ inflateEnd(&zctx);
+ zinit = -1;
+ } else {
+ warnx("inflate failed %d", rv);
+ return 0;
+ }
+ }
+ len = zctx.next_out - buf;
+ } else {
+ if (amt > chunksize)
+ amt = chunksize;
+ len = ftp_read(fin, ssl, buf, amt);
+ chunksize -= len;
+ if (chunksize == 0) /* eat empty line after chunk */
+ ftp_readline(fin, ssl, &chunkbuflen);
+ }
+
+ return len;
+}
+#endif /* SMALL */
+
+
/*
* Retrieve URL, via the proxy in $proxyvar if necessary.
* Modifies the string argument given.
@@ -195,6 +264,8 @@
const char *scheme;
int ishttpsurl = 0;
SSL_CTX *ssl_ctx = NULL;
+ int chunked = 0;
+ int gzipped = 1;
#endif /* !SMALL */
SSL *ssl = NULL;
int status;
@@ -607,9 +678,11 @@
#endif /* !SMALL */
ftp_printf(fin, ssl, "GET /%s %s\r\nHost: ", epath,
#ifndef SMALL
- restart_point ? "HTTP/1.1\r\nConnection: close" :
+ "HTTP/1.1\r\nConnection: close"
+#else
+ "HTTP/1.0"
#endif /* !SMALL */
- "HTTP/1.0");
+ );
if (strchr(host, ':')) {
char *h, *p;
@@ -638,6 +711,7 @@
if (restart_point)
ftp_printf(fin, ssl, "\r\nRange: bytes=%lld-",
(long long)restart_point);
+ ftp_printf(fin, ssl, "\r\nAccept-Encoding: gzip");
#else /* !SMALL */
if (port && strcmp(port, "80") != 0)
ftp_printf(fin, ssl, ":%s", port);
@@ -753,6 +827,17 @@
#ifndef SMALL
if (restart_point)
filesize += restart_point;
+#define TRANSENC "Transfer-Encoding: "
+ } else if (strncasecmp(cp, TRANSENC, sizeof(TRANSENC)-1) == 0) {
+ cp += sizeof(TRANSENC) - 1;
+ if (strcasecmp(cp, "chunked") == 0)
+ chunked = 1;
+#define CONTENTENC "Content-Encoding: "
+ } else if (strncasecmp(cp, CONTENTENC, sizeof(CONTENTENC) -1 )
+ == 0) {
+ cp += sizeof(CONTENTENC) - 1;
+ if (strcasecmp(cp, "gzip") == 0)
+ gzipped = 1;
#endif /* !SMALL */
#define LOCATION "Location: "
} else if (isredirect &&
@@ -853,7 +938,12 @@
len = 1;
oldinti = signal(SIGINFO, psummary);
while (len > 0) {
- len = ftp_read(fin, ssl, buf, 4096);
+#ifndef SMALL
+ if (chunked)
+ len = chunked_read(fin, ssl, buf, 4096, gzipped);
+ else
+#endif
+ len = ftp_read(fin, ssl, buf, 4096);
bytes += len;
for (cp = buf, wlen = len; wlen > 0; wlen -= i, cp += i) {
if ((i = write(out, cp, wlen)) == -1) {
