--- varnish-2.0.1-original/bin/varnishd/cache_fetch.c	2008-10-17 11:59:49.000000000 -0700
+++ varnish-2.0.1/bin/varnishd/cache_fetch.c	2008-11-13 16:19:31.000000000 -0800
@@ -45,14 +45,22 @@
 static unsigned fetchfrag;
 
 /*--------------------------------------------------------------------*/
-
+/*
+ * Read a body with a known content-length.  If 'obj' is non-NULL,
+ * it points to the struct object where the body should be stored.
+ * If 'work' is non-NULL, it points to the worker to which the body
+ * should be written.  If neither is specified, the body is read
+ * and discarded.
+ */
 static int
-fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
+fetch_straight(struct sess *sp, struct http_conn *htc, const char *b,
+	       struct object *obj, struct worker *wrk)
 {
+	unsigned char buf[8192];
 	int i;
 	unsigned char *p;
 	uintmax_t cll;
-	unsigned cl;
+	unsigned cl, bl;
 	struct storage *st;
 
 	cll = strtoumax(b, NULL, 0);
@@ -62,140 +70,174 @@ fetch_straight(struct sess *sp, struct h
 	cl = (unsigned)cll;
 	assert((uintmax_t)cl == cll); /* Protect against bogusly large values */
 
-	st = STV_alloc(sp, cl);
-	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
-	st->len = cl;
-	sp->obj->len = cl;
-	p = st->ptr;
+	if (obj) {
+		/* If we're storing, allocate storage. */
+		st = STV_alloc(sp, cl);
+		VTAILQ_INSERT_TAIL(&obj->store, st, list);
+		st->len = cl;
+		obj->len = cl;
+		p = st->ptr;
+		bl = cl;
+	}
 
+	i = 0;
 	while (cl > 0) {
-		i = HTC_Read(htc, p, cl);
+		if (obj) {
+			p += i;
+			i = cl;
+		} else {
+			/* We're not storing, reuse the stack buffer. */
+			p = buf;
+			if (cl > sizeof buf)
+				i = sizeof buf;
+			else
+				i = cl;
+		}
+		i = HTC_Read(htc, p, i);
 		if (i <= 0)
 			return (-1);
-		p += i;
+		if (wrk) {
+			WRK_Write(wrk, p, i);
+			if (WRK_Flush(wrk))
+				return (2);
+		}
 		cl -= i;
 	}
 	return (0);
 }
 
 /*--------------------------------------------------------------------*/
-/* XXX: Cleanup.  It must be possible somehow :-( */
+/* XXX With a little work, this could be generalized to pass through
+ * chunked encoding from an HTTP/1.1 agent to non-chunked encoding to
+ * an HTTP/1.0 agent.  Just drop the chunk size and end-of-data
+ * markers and close the stream at the end.  Such a change would
+ * require some additional work at higher levels to detect such a
+ * version mismatch and pass the necessary data down to this level.
+ */
 
 static int
-fetch_chunked(struct sess *sp, struct http_conn *htc)
+fetch_chunked(struct sess *sp, struct http_conn *htc, struct object *obj,
+	struct worker *wrk)
 {
+	char buf[8192];		/* XXX: arbitrary */
 	int i;
-	char *q;
 	struct storage *st;
-	unsigned u, v, w;
-	char buf[20];		/* XXX: arbitrary */
-	char *bp, *be;
+	unsigned chunk_length, avail;
+	char *bp, *be, *q;
+	unsigned char *p;
+	unsigned long min_chunksize = params->fetch_chunksize * 1024;
 
 	be = buf + sizeof buf - 1;
-	bp = buf;
 	st = NULL;
-	u = 0;
+	avail = 0;
 	while (1) {
-		/* Try to parse buf as a chunk length */
-		*bp = '\0';
-		u = strtoul(buf, &q, 16);
-
-		/* Skip trailing whitespace */
-		if (q != NULL && q > buf) {
-			while (*q == '\t' || *q == ' ')
-				q++;
-			if (*q == '\r')
-				q++;
-		}
-
-		/* If we didn't succeed, add to buffer, try again */
-		if (q == NULL || q == buf || *q != '\n') {
-			xxxassert(be > bp);
-			/*
-			 * The semantics we need here is "read until you have
-			 * received at least one character, but feel free to
-			 * return up to (be-bp) if they are available, but do
-			 * not wait for those extra characters.
-			 *
-			 * The canonical way to do that is to do a blocking
-			 * read(2) of one char, then change to nonblocking,
-			 * read as many as we find, then change back to
-			 * blocking reads again.
-			 *
-			 * Hardly much more efficient and certainly a good
-			 * deal more complex than reading a single character
-			 * at a time.
-			 */
+		/* Read the chunk header. */
+		/* This is short, so it's okay to read one byte at a time. */
+		bp = buf;
+		do {
 			i = HTC_Read(htc, bp, 1);
 			if (i <= 0)
 				return (-1);
 			bp += i;
-			continue;
+			if (bp >= be)
+				return (-1); /* Malformed (too big) header. */
+		} while (bp[-1] != '\n');
+
+		/* Header format is: size ( ';' extension )* CR LF */
+		*bp = '\0';
+		chunk_length = strtoul(buf, &q, 16);
+		if (chunk_length == ULONG_MAX)
+			return (-1); /* length overflow */
+		if (q == buf)
+			return (-1); /* No length field. */
+		if (*q == ';') {
+			/* Chunk extensions. */
+			/* XXX Support these someday.  For now, just skip. */
+			while (q < bp && *q != '\r' && *q != '\n')
+				++q;
 		}
+		if (*q == '\r') /* N.B. In RFC2616, '\r' is not optional. */
+			++q;
+		if (*q != '\n')
+			return (-1);
 
-		/* Skip NL */
-		q++;
+		/* If we're writing, write the chunk header through. */
+		if (wrk) {
+			WRK_Write(wrk, buf, bp - buf);
+			if (WRK_Flush(wrk))
+				return (2);
+		}
 
 		/* Last chunk is zero bytes long */
-		if (u == 0)
+		if (chunk_length == 0)
 			break;
 
-		while (u > 0) {
-
+		/* Now read the chunk body. */
+		while (chunk_length > 0) {
 			/* Get some storage if we don't have any */
-			if (st == NULL || st->len == st->space) {
-				v = u;
-				if (u < params->fetch_chunksize * 1024)
-					v = params->fetch_chunksize * 1024;
-				st = STV_alloc(sp, v);
-				VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+			if (avail == 0) {
+				if (obj) {
+					avail = chunk_length;
+					if (avail < min_chunksize)
+						avail = min_chunksize;
+					st = STV_alloc(sp, avail);
+					VTAILQ_INSERT_TAIL(&obj->store,
+							   st, list);
+					p = st->ptr;
+				} else {
+					avail = sizeof buf;
+					p = (unsigned char *)buf;
+				}
 			}
-			v = st->space - st->len;
-			if (v > u)
-				v = u;
-
-			/* Handle anything left in our buffer first */
-			w = pdiff(q, bp);
-			if (w > v)
-				w = v;
-			if (w != 0) {
-				memcpy(st->ptr + st->len, q, w);
-				st->len += w;
-				sp->obj->len += w;
-				u -= w;
-				v -= w;
-				q += w;
-			}
-			if (u == 0)
-				break;
-			if (v == 0)
-				continue;
 
-			/* Pick up the rest of this chunk */
-			while (v > 0) {
-				i = HTC_Read(htc, st->ptr + st->len, v);
+			/* Read data until we've finished the chunk
+			 * or filled the next storage block. */
+			while (avail > 0 && chunk_length > 0) {
+				if (avail < chunk_length)
+					i = avail;
+				else
+					i = chunk_length;
+				i = HTC_Read(htc, p, i);
 				if (i <= 0)
 					return (-1);
-				st->len += i;
-				sp->obj->len += i;
-				u -= i;
-				v -= i;
+				if (wrk) {
+					WRK_Write(wrk, p, i);
+					if (WRK_Flush(wrk)) {
+						return (2);
+					}
+				}
+				if (obj) {
+					st->len += i;
+					obj->len += i;
+				}
+				chunk_length -= i;
+				avail -= i;
+				p += i;
 			}
 		}
-		assert(u == 0);
+		assert(chunk_length == 0);
 
-		/* We might still have stuff in our buffer */
-		v = pdiff(q, bp);
-		if (v > 0)
-			memcpy(buf, q, v);
-		q = bp = buf + v;
+		/* Chunk trailer is CR LF. */
+		bp = buf;
+		do {
+			i = HTC_Read(htc, bp, 1);
+			if (i <= 0)
+				return (-1);
+			bp += i;
+			xxxassert(be > bp);
+		} while (bp[-1] != '\n');
+		if (wrk)
+			WRK_Write(wrk, buf, bp - buf);
 	}
 
-	if (st != NULL && st->len == 0) {
-		VTAILQ_REMOVE(&sp->obj->store, st, list);
-		STV_free(st);
-	} else if (st != NULL)
-		STV_trim(st, st->len);
+	/* If we stored the data, clean up the last element in the list. */
+	if (obj) {
+		if (st != NULL && st->len == 0) {
+			VTAILQ_REMOVE(&sp->obj->store, st, list);
+			STV_free(st);
+		} else if (st != NULL)
+			STV_trim(st, st->len);
+	}
 	return (0);
 }
 
@@ -268,34 +310,24 @@ fetch_eof(struct sess *sp, struct http_c
 int
 FetchReqBody(struct sess *sp)
 {
-	unsigned long content_length;
-	char buf[8192];
-	char *ptr, *endp;
-	int rdcnt;
-
-	if (http_GetHdr(sp->http, H_Content_Length, &ptr)) {
-
-		content_length = strtoul(ptr, &endp, 10);
-		/* XXX should check result of conversion */
-		while (content_length) {
-			if (content_length > sizeof buf)
-				rdcnt = sizeof buf;
-			else
-				rdcnt = content_length;
-			rdcnt = HTC_Read(sp->htc, buf, rdcnt);
-			if (rdcnt <= 0)
-				return (1);
-			content_length -= rdcnt;
-			if (!sp->sendbody)
-				continue;
-			WRK_Write(sp->wrk, buf, rdcnt);	/* XXX: stats ? */
-			if (WRK_Flush(sp->wrk))
-				return (2);
-		}
-	}
-	if (http_GetHdr(sp->http, H_Transfer_Encoding, NULL)) {
-		/* XXX: Handle chunked encoding. */
-		WSL(sp->wrk, SLT_Debug, sp->fd, "Transfer-Encoding in request");
+	char *b;
+	int r;
+
+	/* TODO: We should consider the request type here as well:
+	 * GET, HEAD, and a few others are not allowed to have bodies. */
+	if (http_GetHdr(sp->http, H_Content_Length, &b)) {
+		r = fetch_straight(sp, sp->htc, b, NULL,
+				   sp->sendbody ? sp->wrk : NULL);
+		if (r < 0)
+			return (1);
+		if (r == 2)
+			return (2);
+	} else if (http_HdrIs(sp->http, H_Transfer_Encoding, "chunked")) {
+		return fetch_chunked(sp, sp->htc, NULL,
+				   sp->sendbody ? sp->wrk : NULL);
+	} else if (http_GetHdr(sp->http, H_Transfer_Encoding, &b)) {
+		WSL(sp->wrk, SLT_Debug, sp->fd,
+		    "Invalid Transfer-Encoding in request");
 		return (1);
 	}
 	return (0);
@@ -404,10 +436,10 @@ Fetch(struct sess *sp)
 	if (is_head) {
 		/* nothing */
 	} else if (http_GetHdr(hp, H_Content_Length, &b)) {
-		cls = fetch_straight(sp, htc, b);
+		cls = fetch_straight(sp, htc, b, sp->obj, NULL);
 		mklen = 1;
 	} else if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) {
-		cls = fetch_chunked(sp, htc);
+		cls = fetch_chunked(sp, htc, sp->obj, NULL);
 		mklen = 1;
 	} else if (http_GetHdr(hp, H_Transfer_Encoding, &b)) {
 		/* XXX: AUGH! */