On Sun, 9 Jan 2011, Ted Unangst wrote:
> Downloading things can go a lot faster if the server and client support
> http compression. This is easily added to the ftp program's http support.
>
> It consists of two parts. Support for deflating the data we receive and
> support for the chunked transfer the server will use to send data to us.
> The chunked supported is probably useful in its own right as well.
Upon further reflection, I realize it's possible for a server to send
gzipped data that's not chunked. The diff below is a better version that
should be able to handle all combinations of chunked, gzipped, both, or
none.
Index: Makefile
===================================================================
RCS file: /home/tedu/cvs/src/usr.bin/ftp/Makefile,v
retrieving revision 1.25
diff -u -r1.25 Makefile
--- Makefile 5 May 2009 19:35:30 -0000 1.25
+++ Makefile 9 Jan 2011 21:14:51 -0000
@@ -17,8 +17,8 @@
CPPFLAGS+= -DINET6
-LDADD+= -ledit -lcurses -lutil -lssl -lcrypto
-DPADD+= ${LIBEDIT} ${LIBCURSES} ${LIBUTIL}
+LDADD+= -ledit -lcurses -lutil -lssl -lcrypto -lz
+DPADD+= ${LIBEDIT} ${LIBCURSES} ${LIBUTIL} ${LIBZ}
LDSTATIC= ${STATIC}
#COPTS+= -Wall -Wconversion -Wstrict-prototypes -Wmissing-prototypes
Index: fetch.c
===================================================================
RCS file: /home/tedu/cvs/src/usr.bin/ftp/fetch.c,v
retrieving revision 1.103
diff -u -r1.103 fetch.c
--- fetch.c 25 Aug 2010 20:32:37 -0000 1.103
+++ fetch.c 9 Jan 2011 21:38:27 -0000
@@ -63,6 +63,7 @@
#ifndef SMALL
#include <openssl/ssl.h>
#include <openssl/err.h>
+#include <zlib.h>
#else /* !SMALL */
#define SSL void
#endif /* !SMALL */
@@ -167,6 +168,80 @@
return (epath);
}
+#ifndef SMALL
+static size_t
+chunked_read(FILE *fin, SSL *ssl, unsigned char *buf, size_t amt,
+ int chunked, int gzipped)
+{
+ static int chunksize;
+ static int zinit;
+ static z_stream zctx;
+ static unsigned char zbuf[4096];
+
+ size_t chunkbuflen;
+ char *chunkbuf;
+ size_t len, zlen, zamt;
+ int rv;
+
+ if (chunked && !chunksize) {
+ chunkbuf = ftp_readline(fin, ssl, &chunkbuflen);
+ if (!chunkbuf) {
+ warnx("no chunk size");
+ return 0;
+ }
+ chunksize = strtol(chunkbuf, NULL, 16);
+ }
+ if (gzipped) {
+ if (zinit == -1) {
+ zinit = 0;
+ return 0;
+ }
+ if (zctx.avail_in == 0) {
+ zamt = sizeof(zbuf);
+ if (chunked && zamt > chunksize)
+ zamt = chunksize;
+ zlen = ftp_read(fin, ssl, zbuf, zamt);
+ if (chunked) {
+ chunksize -= zlen;
+ /* eat empty line after chunk */
+ if (chunksize == 0)
+ ftp_readline(fin, ssl, &chunkbuflen);
+ }
+ zctx.avail_in = zlen;
+ zctx.next_in = zbuf;
+ }
+ if (!zinit) {
+ rv = inflateInit2(&zctx, 15 + 32);
+ zinit = 1;
+ }
+ zctx.next_out = buf;
+ zctx.avail_out = amt;
+ if ((rv = inflate(&zctx, Z_NO_FLUSH)) != Z_OK) {
+ if (rv == Z_STREAM_END) {
+ inflateEnd(&zctx);
+ zinit = -1;
+ } else {
+ warnx("inflate failed %d", rv);
+ return 0;
+ }
+ }
+ len = zctx.next_out - buf;
+ } else if (chunked) {
+ if (amt > chunksize)
+ amt = chunksize;
+ len = ftp_read(fin, ssl, buf, amt);
+ chunksize -= len;
+ if (chunksize == 0) /* eat empty line after chunk */
+ ftp_readline(fin, ssl, &chunkbuflen);
+ } else {
+ len = ftp_read(fin, ssl, buf, amt);
+ }
+
+ return len;
+}
+#endif /* SMALL */
+
+
/*
* Retrieve URL, via the proxy in $proxyvar if necessary.
* Modifies the string argument given.
@@ -195,6 +270,8 @@
const char *scheme;
int ishttpsurl = 0;
SSL_CTX *ssl_ctx = NULL;
+ int chunked = 0;
+ int gzipped = 1;
#endif /* !SMALL */
SSL *ssl = NULL;
int status;
@@ -607,9 +684,11 @@
#endif /* !SMALL */
ftp_printf(fin, ssl, "GET /%s %s\r\nHost: ", epath,
#ifndef SMALL
- restart_point ? "HTTP/1.1\r\nConnection: close" :
+ "HTTP/1.1\r\nConnection: close"
+#else
+ "HTTP/1.0"
#endif /* !SMALL */
- "HTTP/1.0");
+ );
if (strchr(host, ':')) {
char *h, *p;
@@ -638,6 +717,7 @@
if (restart_point)
ftp_printf(fin, ssl, "\r\nRange: bytes=%lld-",
(long long)restart_point);
+ ftp_printf(fin, ssl, "\r\nAccept-Encoding: gzip");
#else /* !SMALL */
if (port && strcmp(port, "80") != 0)
ftp_printf(fin, ssl, ":%s", port);
@@ -753,6 +833,17 @@
#ifndef SMALL
if (restart_point)
filesize += restart_point;
+#define TRANSENC "Transfer-Encoding: "
+ } else if (strncasecmp(cp, TRANSENC, sizeof(TRANSENC)-1) == 0) {
+ cp += sizeof(TRANSENC) - 1;
+ if (strcasecmp(cp, "chunked") == 0)
+ chunked = 1;
+#define CONTENTENC "Content-Encoding: "
+ } else if (strncasecmp(cp, CONTENTENC, sizeof(CONTENTENC) -1 )
+ == 0) {
+ cp += sizeof(CONTENTENC) - 1;
+ if (strcasecmp(cp, "gzip") == 0)
+ gzipped = 1;
#endif /* !SMALL */
#define LOCATION "Location: "
} else if (isredirect &&
@@ -853,7 +944,13 @@
len = 1;
oldinti = signal(SIGINFO, psummary);
while (len > 0) {
- len = ftp_read(fin, ssl, buf, 4096);
+#ifndef SMALL
+ if (chunked || gzipped)
+ len = chunked_read(fin, ssl, buf, 4096,
+ chunked, gzipped);
+ else
+#endif
+ len = ftp_read(fin, ssl, buf, 4096);
bytes += len;
for (cp = buf, wlen = len; wlen > 0; wlen -= i, cp += i) {
if ((i = write(out, cp, wlen)) == -1) {
