Module Name: src Committed By: snj Date: Mon Feb 15 00:47:56 UTC 2010
Modified Files: src/external/bsd/fetch/dist/libfetch [netbsd-5-0]: common.c common.h errlist.sh fetch.3 fetch.c fetch.cat3 fetch.h file.c ftp.c ftp.errors http.c http.errors Log Message: Apply patch (requested by joerg in ticket #1294): Update libfetch to 2.30. Changes: libfetch-2.25: - address a number of lint warnings - fix strict-alignment issues for GCC 4.4 - fix a bug in the line reading optimisation - do not reuse a FTP connection if there is an active transfer on it libfetch-2.26: - Add support to aggressively cache directory listenings; useful for HTTP - Avoid leaking memory in error cases. From Xavier from Arch Linux. libfetch-2.30: - Revamped connection cache, allowing more than one active session - HTTP keep-alive support To generate a diff of this commit: cvs rdiff -u -r1.1.1.2.8.1 -r1.1.1.2.8.2 \ src/external/bsd/fetch/dist/libfetch/common.c \ src/external/bsd/fetch/dist/libfetch/common.h \ src/external/bsd/fetch/dist/libfetch/errlist.sh \ src/external/bsd/fetch/dist/libfetch/fetch.h \ src/external/bsd/fetch/dist/libfetch/file.c \ src/external/bsd/fetch/dist/libfetch/ftp.errors \ src/external/bsd/fetch/dist/libfetch/http.c \ src/external/bsd/fetch/dist/libfetch/http.errors cvs rdiff -u -r1.1.1.3.6.1 -r1.1.1.3.6.2 \ src/external/bsd/fetch/dist/libfetch/fetch.3 \ src/external/bsd/fetch/dist/libfetch/ftp.c cvs rdiff -u -r1.1.1.3.4.1 -r1.1.1.3.4.2 \ src/external/bsd/fetch/dist/libfetch/fetch.c cvs rdiff -u -r1.1.1.2.6.1 -r1.1.1.2.6.2 \ src/external/bsd/fetch/dist/libfetch/fetch.cat3 Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/external/bsd/fetch/dist/libfetch/common.c diff -u src/external/bsd/fetch/dist/libfetch/common.c:1.1.1.2.8.1 src/external/bsd/fetch/dist/libfetch/common.c:1.1.1.2.8.2 --- src/external/bsd/fetch/dist/libfetch/common.c:1.1.1.2.8.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/common.c Mon Feb 15 00:47:55 2010 @@ -1,7 +1,7 @@ -/* $NetBSD: common.c,v 1.1.1.2.8.1 2009/05/30 15:58:17 snj Exp $ */ +/* $NetBSD: common.c,v 1.1.1.2.8.2 2010/02/15 00:47:55 snj Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav - * Copyright (c) 2008 Joerg Sonnenberger <jo...@netbsd.org> + * Copyright (c) 2008, 2010 Joerg Sonnenberger <jo...@netbsd.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -62,12 +62,13 @@ #include <string.h> #include <unistd.h> +#ifndef MSG_NOSIGNAL +#include <signal.h> +#endif + #include "fetch.h" #include "common.h" -#define DECONST(x,y) ((x)(uintptr_t)(y)) - - /*** Local data **************************************************************/ /* @@ -83,10 +84,6 @@ { -1, FETCH_UNKNOWN, "Unknown resolver error" } }; -/* End-of-Line */ -static const char ENDL[2] = "\r\n"; - - /*** Error-reporting functions ***********************************************/ /* @@ -234,22 +231,10 @@ /* allocate and fill connection structure */ if ((conn = calloc(1, sizeof(*conn))) == NULL) return (NULL); + conn->cache_url = NULL; conn->next_buf = NULL; conn->next_len = 0; conn->sd = sd; - ++conn->ref; - return (conn); -} - - -/* - * Bump a connection's reference count. - */ -conn_t * -fetch_ref(conn_t *conn) -{ - - ++conn->ref; return (conn); } @@ -261,17 +246,17 @@ fetch_bind(int sd, int af, const char *addr) { struct addrinfo hints, *res, *res0; - int error; memset(&hints, 0, sizeof(hints)); hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = 0; - if ((error = getaddrinfo(addr, NULL, &hints, &res0)) != 0) + if (getaddrinfo(addr, NULL, &hints, &res0)) return (-1); - for (res = res0; res; res = res->ai_next) + for (res = res0; res; res = res->ai_next) { if (bind(sd, res->ai_addr, res->ai_addrlen) == 0) return (0); + } return (-1); } @@ -280,7 +265,7 @@ * Establish a TCP connection to the specified port on the specified host. */ conn_t * -fetch_connect(const char *host, int port, int af, int verbose) +fetch_connect(struct url *url, int af, int verbose) { conn_t *conn; char pbuf[10]; @@ -289,22 +274,22 @@ int sd, error; if (verbose) - fetch_info("looking up %s", host); + fetch_info("looking up %s", url->host); /* look up host name and set up socket address structure */ - snprintf(pbuf, sizeof(pbuf), "%d", port); + snprintf(pbuf, sizeof(pbuf), "%d", url->port); memset(&hints, 0, sizeof(hints)); hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = 0; - if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) { + if ((error = getaddrinfo(url->host, pbuf, &hints, &res0)) != 0) { netdb_seterr(error); return (NULL); } bindaddr = getenv("FETCH_BIND_ADDRESS"); if (verbose) - fetch_info("connecting to %s:%d", host, port); + fetch_info("connecting to %s:%d", url->host, url->port); /* try to connect */ for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) { @@ -331,9 +316,114 @@ fetch_syserr(); close(sd); } + conn->cache_url = fetchCopyURL(url); + conn->cache_af = af; return (conn); } +static conn_t *connection_cache; +static int cache_global_limit = 0; +static int cache_per_host_limit = 0; + +/* + * Initialise cache with the given limits. + */ +void +fetchConnectionCacheInit(int global_limit, int per_host_limit) +{ + + if (global_limit < 0) + cache_global_limit = INT_MAX; + else if (per_host_limit > global_limit) + cache_global_limit = per_host_limit; + else + cache_global_limit = global_limit; + if (per_host_limit < 0) + cache_per_host_limit = INT_MAX; + else + cache_per_host_limit = per_host_limit; +} + +/* + * Flush cache and free all associated resources. + */ +void +fetchConnectionCacheClose(void) +{ + conn_t *conn; + + while ((conn = connection_cache) != NULL) { + connection_cache = conn->next_cached; + (*conn->cache_close)(conn); + } +} + +/* + * Check connection cache for an existing entry matching + * protocol/host/port/user/password/family. + */ +conn_t * +fetch_cache_get(const struct url *url, int af) +{ + conn_t *conn, *last_conn = NULL; + + for (conn = connection_cache; conn; conn = conn->next_cached) { + if (conn->cache_url->port == url->port && + strcmp(conn->cache_url->scheme, url->scheme) == 0 && + strcmp(conn->cache_url->host, url->host) == 0 && + strcmp(conn->cache_url->user, url->user) == 0 && + strcmp(conn->cache_url->pwd, url->pwd) == 0 && + (conn->cache_af == AF_UNSPEC || af == AF_UNSPEC || + conn->cache_af == af)) { + if (last_conn != NULL) + last_conn->next_cached = conn->next_cached; + else + connection_cache = conn->next_cached; + return conn; + } + } + + return NULL; +} + +/* + * Put the connection back into the cache for reuse. + * If the connection is freed due to LRU or if the cache + * is explicitly closed, the given callback is called. + */ +void +fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *)) +{ + conn_t *iter, *last; + int global_count, host_count; + + if (conn->cache_url == NULL || cache_global_limit == 0) { + (*closecb)(conn); + return; + } + + global_count = host_count = 0; + last = NULL; + for (iter = connection_cache; iter; + last = iter, iter = iter->next_cached) { + ++global_count; + if (strcmp(conn->cache_url->host, iter->cache_url->host) == 0) + ++host_count; + if (global_count < cache_global_limit && + host_count < cache_per_host_limit) + continue; + --global_count; + if (last != NULL) + last->next_cached = iter->next_cached; + else + connection_cache = iter->next_cached; + (*iter->cache_close)(iter); + } + + conn->cache_close = closecb; + conn->next_cached = connection_cache; + connection_cache = conn; +} /* * Enable SSL on a connection. @@ -475,18 +565,22 @@ ssize_t len; if (conn->buf == NULL) { - if ((conn->buf = malloc(MIN_BUF_SIZE + 1)) == NULL) { + if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) { errno = ENOMEM; return (-1); } conn->bufsize = MIN_BUF_SIZE; } - conn->buf[0] = '\0'; conn->buflen = 0; next = NULL; do { + /* + * conn->bufsize != conn->buflen at this point, + * so the buffer can be NUL-terminated below for + * the case of len == 0. + */ len = fetch_read(conn, conn->buf + conn->buflen, conn->bufsize - conn->buflen); if (len == -1) @@ -495,10 +589,13 @@ break; next = memchr(conn->buf + conn->buflen, '\n', len); conn->buflen += len; - if (conn->buflen == conn->bufsize && - (next == NULL || next[1] == '\0')) { + if (conn->buflen == conn->bufsize && next == NULL) { tmp = conn->buf; - tmpsize = conn->bufsize * 2 + 1; + tmpsize = conn->bufsize * 2; + if (tmpsize < conn->bufsize) { + errno = ENOMEM; + return (-1); + } if ((tmp = realloc(tmp, tmpsize)) == NULL) { errno = ENOMEM; return (-1); @@ -509,39 +606,39 @@ } while (next == NULL); if (next != NULL) { + *next = '\0'; conn->next_buf = next + 1; conn->next_len = conn->buflen - (conn->next_buf - conn->buf); conn->buflen = next - conn->buf; + } else { + conn->buf[conn->buflen] = '\0'; + conn->next_len = 0; } - conn->buf[conn->buflen] = '\0'; return (0); } - -/* - * Write to a connection w/ timeout - */ -ssize_t -fetch_write(conn_t *conn, const char *buf, size_t len) -{ - struct iovec iov; - - iov.iov_base = DECONST(char *, buf); - iov.iov_len = len; - return fetch_writev(conn, &iov, 1); -} - /* * Write a vector to a connection w/ timeout * Note: can modify the iovec. */ ssize_t -fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) +fetch_write(conn_t *conn, const void *buf, size_t len) { struct timeval now, timeout, waittv; fd_set writefds; ssize_t wlen, total; int r; +#ifndef MSG_NOSIGNAL + static int killed_sigpipe; +#endif + +#ifndef MSG_NOSIGNAL + if (!killed_sigpipe) { + signal(SIGPIPE, SIG_IGN); + killed_sigpipe = 1; + } +#endif + if (fetchTimeout) { FD_ZERO(&writefds); @@ -550,7 +647,7 @@ } total = 0; - while (iovcnt > 0) { + while (len) { while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) { FD_SET(conn->sd, &writefds); gettimeofday(&now, NULL); @@ -576,11 +673,14 @@ errno = 0; #ifdef WITH_SSL if (conn->ssl != NULL) - wlen = SSL_write(conn->ssl, - iov->iov_base, iov->iov_len); + wlen = SSL_write(conn->ssl, buf, len); else #endif - wlen = writev(conn->sd, iov, iovcnt); +#ifndef MSG_NOSIGNAL + wlen = send(conn->sd, buf, len, 0); +#else + wlen = send(conn->sd, buf, len, MSG_NOSIGNAL); +#endif if (wlen == 0) { /* we consider a short write a failure */ errno = EPIPE; @@ -593,44 +693,14 @@ return (-1); } total += wlen; - while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) { - wlen -= iov->iov_len; - iov++; - iovcnt--; - } - if (iovcnt > 0) { - iov->iov_len -= wlen; - iov->iov_base = DECONST(char *, iov->iov_base) + wlen; - } + buf = (const char *)buf + wlen; + len -= wlen; } return (total); } /* - * Write a line of text to a connection w/ timeout - */ -int -fetch_putln(conn_t *conn, const char *str, size_t len) -{ - struct iovec iov[2]; - int ret; - - iov[0].iov_base = DECONST(char *, str); - iov[0].iov_len = len; - iov[1].iov_base = DECONST(char *, ENDL); - iov[1].iov_len = sizeof(ENDL); - if (len == 0) - ret = fetch_writev(conn, &iov[1], 1); - else - ret = fetch_writev(conn, iov, 2); - if (ret == -1) - return (-1); - return (0); -} - - -/* * Close connection */ int @@ -638,9 +708,9 @@ { int ret; - if (--conn->ref > 0) - return (0); ret = close(conn->sd); + if (conn->cache_url) + fetchFreeURL(conn->cache_url); free(conn->buf); free(conn); return (ret); @@ -742,6 +812,40 @@ ue->urls = NULL; } +int +fetchAppendURLList(struct url_list *dst, const struct url_list *src) +{ + size_t i, j, len; + + len = dst->length + src->length; + if (len > dst->alloc_size) { + struct url *tmp; + + tmp = realloc(dst->urls, len * sizeof(*tmp)); + if (tmp == NULL) { + errno = ENOMEM; + fetch_syserr(); + return (-1); + } + dst->alloc_size = len; + dst->urls = tmp; + } + + for (i = 0, j = dst->length; i < src->length; ++i, ++j) { + dst->urls[j] = src->urls[i]; + dst->urls[j].doc = strdup(src->urls[i].doc); + if (dst->urls[j].doc == NULL) { + while (i-- > 0) + free(dst->urls[j].doc); + fetch_syserr(); + return -1; + } + } + dst->length = len; + + return 0; +} + void fetchFreeURLList(struct url_list *ue) { Index: src/external/bsd/fetch/dist/libfetch/common.h diff -u src/external/bsd/fetch/dist/libfetch/common.h:1.1.1.2.8.1 src/external/bsd/fetch/dist/libfetch/common.h:1.1.1.2.8.2 --- src/external/bsd/fetch/dist/libfetch/common.h:1.1.1.2.8.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/common.h Mon Feb 15 00:47:55 2010 @@ -1,4 +1,4 @@ -/* $NetBSD: common.h,v 1.1.1.2.8.1 2009/05/30 15:58:17 snj Exp $ */ +/* $NetBSD: common.h,v 1.1.1.2.8.2 2010/02/15 00:47:55 snj Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * All rights reserved. @@ -53,6 +53,7 @@ /* Connection */ typedef struct fetchconn conn_t; + struct fetchconn { int sd; /* socket descriptor */ char *buf; /* buffer */ @@ -71,7 +72,11 @@ const SSL_METHOD *ssl_meth; /* SSL method */ # endif #endif - int ref; /* reference count */ + + struct url *cache_url; + int cache_af; + int (*cache_close)(conn_t *); + conn_t *next_cached; }; /* Structure used for error message lists */ @@ -81,24 +86,20 @@ const char *string; }; -/* for fetch_writev */ -struct iovec; - void fetch_seterr(struct fetcherr *, int); void fetch_syserr(void); void fetch_info(const char *, ...); int fetch_default_port(const char *); int fetch_default_proxy_port(const char *); int fetch_bind(int, int, const char *); -conn_t *fetch_connect(const char *, int, int, int); +conn_t *fetch_cache_get(const struct url *, int); +void fetch_cache_put(conn_t *, int (*)(conn_t *)); +conn_t *fetch_connect(struct url *, int, int); conn_t *fetch_reopen(int); -conn_t *fetch_ref(conn_t *); int fetch_ssl(conn_t *, int); ssize_t fetch_read(conn_t *, char *, size_t); int fetch_getln(conn_t *); -ssize_t fetch_write(conn_t *, const char *, size_t); -ssize_t fetch_writev(conn_t *, struct iovec *, int); -int fetch_putln(conn_t *, const char *, size_t); +ssize_t fetch_write(conn_t *, const void *, size_t); int fetch_close(conn_t *); int fetch_add_entry(struct url_list *, struct url *, const char *, int); int fetch_netrc_auth(struct url *url); Index: src/external/bsd/fetch/dist/libfetch/errlist.sh diff -u src/external/bsd/fetch/dist/libfetch/errlist.sh:1.1.1.2.8.1 src/external/bsd/fetch/dist/libfetch/errlist.sh:1.1.1.2.8.2 --- src/external/bsd/fetch/dist/libfetch/errlist.sh:1.1.1.2.8.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/errlist.sh Mon Feb 15 00:47:55 2010 @@ -1,5 +1,5 @@ #!/bin/sh -# $NetBSD: errlist.sh,v 1.1.1.2.8.1 2009/05/30 15:58:17 snj Exp $ +# $NetBSD: errlist.sh,v 1.1.1.2.8.2 2010/02/15 00:47:55 snj Exp $ printf "static struct fetcherr $1[] = {\n" while read code type msg; do Index: src/external/bsd/fetch/dist/libfetch/fetch.h diff -u src/external/bsd/fetch/dist/libfetch/fetch.h:1.1.1.2.8.1 src/external/bsd/fetch/dist/libfetch/fetch.h:1.1.1.2.8.2 --- src/external/bsd/fetch/dist/libfetch/fetch.h:1.1.1.2.8.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/fetch.h Mon Feb 15 00:47:55 2010 @@ -1,4 +1,4 @@ -/* $NetBSD: fetch.h,v 1.1.1.2.8.1 2009/05/30 15:58:17 snj Exp $ */ +/* $NetBSD: fetch.h,v 1.1.1.2.8.2 2010/02/15 00:47:55 snj Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * All rights reserved. @@ -153,10 +153,15 @@ /* URL listening */ void fetchInitURLList(struct url_list *); +int fetchAppendURLList(struct url_list *, const struct url_list *); void fetchFreeURLList(struct url_list *); char *fetchUnquotePath(struct url *); char *fetchUnquoteFilename(struct url *); +/* Connection caching */ +void fetchConnectionCacheInit(int, int); +void fetchConnectionCacheClose(void); + /* Authentication */ typedef int (*auth_t)(struct url *); extern auth_t fetchAuthMethod; Index: src/external/bsd/fetch/dist/libfetch/file.c diff -u src/external/bsd/fetch/dist/libfetch/file.c:1.1.1.2.8.1 src/external/bsd/fetch/dist/libfetch/file.c:1.1.1.2.8.2 --- src/external/bsd/fetch/dist/libfetch/file.c:1.1.1.2.8.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/file.c Mon Feb 15 00:47:55 2010 @@ -1,4 +1,4 @@ -/* $NetBSD: file.c,v 1.1.1.2.8.1 2009/05/30 15:58:17 snj Exp $ */ +/* $NetBSD: file.c,v 1.1.1.2.8.2 2010/02/15 00:47:55 snj Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * Copyright (c) 2008, 2009 Joerg Sonnenberger <jo...@netbsd.org> @@ -234,6 +234,7 @@ char *path; struct dirent *de; DIR *dir; + int ret; if ((path = fetchUnquotePath(u)) == NULL) { fetch_syserr(); @@ -248,13 +249,17 @@ return -1; } + ret = 0; + while ((de = readdir(dir)) != NULL) { if (pattern && fnmatch(pattern, de->d_name, 0) != 0) continue; - fetch_add_entry(ue, u, de->d_name, 0); + ret = fetch_add_entry(ue, u, de->d_name, 0); + if (ret) + break; } closedir(dir); - return 0; + return ret; } Index: src/external/bsd/fetch/dist/libfetch/ftp.errors diff -u src/external/bsd/fetch/dist/libfetch/ftp.errors:1.1.1.2.8.1 src/external/bsd/fetch/dist/libfetch/ftp.errors:1.1.1.2.8.2 --- src/external/bsd/fetch/dist/libfetch/ftp.errors:1.1.1.2.8.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/ftp.errors Mon Feb 15 00:47:56 2010 @@ -1,4 +1,4 @@ -# $NetBSD: ftp.errors,v 1.1.1.2.8.1 2009/05/30 15:58:17 snj Exp $ +# $NetBSD: ftp.errors,v 1.1.1.2.8.2 2010/02/15 00:47:56 snj Exp $ # $FreeBSD: ftp.errors,v 1.6 2002/10/30 06:06:16 des Exp $ # # This list is taken from RFC 959. Index: src/external/bsd/fetch/dist/libfetch/http.c diff -u src/external/bsd/fetch/dist/libfetch/http.c:1.1.1.2.8.1 src/external/bsd/fetch/dist/libfetch/http.c:1.1.1.2.8.2 --- src/external/bsd/fetch/dist/libfetch/http.c:1.1.1.2.8.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/http.c Mon Feb 15 00:47:56 2010 @@ -1,4 +1,4 @@ -/* $NetBSD: http.c,v 1.1.1.2.8.1 2009/05/30 15:58:17 snj Exp $ */ +/* $NetBSD: http.c,v 1.1.1.2.8.2 2010/02/15 00:47:56 snj Exp $ */ /*- * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav * Copyright (c) 2003 Thomas Klausner <w...@netbsd.org> @@ -142,6 +142,7 @@ { conn_t *conn; /* connection */ int chunked; /* chunked mode */ + int keep_alive; /* keep-alive mode */ char *buf; /* chunk buffer */ size_t bufsize; /* size of chunk buffer */ ssize_t buflen; /* amount of data currently in buffer */ @@ -149,6 +150,7 @@ int eof; /* end-of-file flag */ int error; /* error flag */ size_t chunksize; /* remaining size of current chunk */ + off_t contentlength; /* remaining size of the content */ }; /* @@ -211,6 +213,9 @@ if (io->eof) return (0); + if (io->contentlength >= 0 && (off_t)len > io->contentlength) + len = io->contentlength; + if (io->chunked == 0) { if (http_growbuf(io, len) == -1) return (-1); @@ -218,6 +223,8 @@ io->error = 1; return (-1); } + if (io->contentlength) + io->contentlength -= io->buflen; io->bufpos = 0; return (io->buflen); } @@ -229,6 +236,8 @@ return (-1); case 0: io->eof = 1; + if (fetch_getln(io->conn) == -1) + return (-1); return (0); } } @@ -242,6 +251,8 @@ return (-1); } io->chunksize -= io->buflen; + if (io->contentlength >= 0) + io->contentlength -= io->buflen; if (io->chunksize == 0) { char endl[2]; @@ -309,9 +320,23 @@ { struct httpio *io = (struct httpio *)v; - fetch_close(io->conn); - if (io->buf) - free(io->buf); + if (io->keep_alive) { + int val; + + val = 0; + setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NODELAY, &val, + sizeof(val)); + fetch_cache_put(io->conn, fetch_close); +#ifdef TCP_NOPUSH + val = 1; + setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, + sizeof(val)); +#endif + } else { + fetch_close(io->conn); + } + + free(io->buf); free(io); } @@ -319,7 +344,7 @@ * Wrap a file descriptor up */ static fetchIO * -http_funopen(conn_t *conn, int chunked) +http_funopen(conn_t *conn, int chunked, int keep_alive, off_t clength) { struct httpio *io; fetchIO *f; @@ -330,6 +355,8 @@ } io->conn = conn; io->chunked = chunked; + io->contentlength = clength; + io->keep_alive = keep_alive; f = fetchIO_unopen(io, http_readfn, http_writefn, http_closefn); if (f == NULL) { fetch_syserr(); @@ -350,6 +377,7 @@ hdr_error = -1, hdr_end = 0, hdr_unknown = 1, + hdr_connection, hdr_content_length, hdr_content_range, hdr_last_modified, @@ -363,6 +391,7 @@ hdr_t num; const char *name; } hdr_names[] = { + { hdr_connection, "Connection" }, { hdr_content_length, "Content-Length" }, { hdr_content_range, "Content-Range" }, { hdr_last_modified, "Last-Modified" }, @@ -393,7 +422,7 @@ return (-1); } - r = fetch_putln(conn, msg, len); + r = fetch_write(conn, msg, len); free(msg); if (r == -1) { @@ -634,7 +663,7 @@ free(upw); if (auth == NULL) return (-1); - r = http_cmd(conn, "%s: Basic %s", hdr, auth); + r = http_cmd(conn, "%s: Basic %s\r\n", hdr, auth); free(auth); return (r); } @@ -676,7 +705,7 @@ * Connect to the correct HTTP server or proxy. */ static conn_t * -http_connect(struct url *URL, struct url *purl, const char *flags) +http_connect(struct url *URL, struct url *purl, const char *flags, int *cached) { conn_t *conn; int af, verbose; @@ -684,6 +713,8 @@ int val; #endif + *cached = 1; + #ifdef INET6 af = AF_UNSPEC; #else @@ -706,7 +737,12 @@ return (NULL); } - if ((conn = fetch_connect(URL->host, URL->port, af, verbose)) == NULL) + if ((conn = fetch_cache_get(URL, af)) != NULL) { + *cached = 1; + return (conn); + } + + if ((conn = fetch_connect(URL, af, verbose)) == NULL) /* fetch_connect() has already set an error code */ return (NULL); if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && @@ -764,7 +800,7 @@ snprintf(buf, sizeof(buf), "%.3s, %02d %.3s %4d %02d:%02d:%02d GMT", weekdays + tm.tm_wday * 3, tm.tm_mday, months + tm.tm_mon * 3, tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec); - http_cmd(conn, "If-Modified-Since: %s", buf); + http_cmd(conn, "If-Modified-Since: %s\r\n", buf); } @@ -784,7 +820,8 @@ { conn_t *conn; struct url *url, *new; - int chunked, direct, if_modified_since, need_auth, noredirect, verbose; + int chunked, direct, if_modified_since, need_auth, noredirect; + int keep_alive, verbose, cached; int e, i, n, val; off_t offset, clength, length, size; time_t mtime; @@ -797,6 +834,7 @@ noredirect = CHECK_FLAG('A'); verbose = CHECK_FLAG('v'); if_modified_since = CHECK_FLAG('i'); + keep_alive = 0; if (direct && purl) { fetchFreeURL(purl); @@ -834,7 +872,7 @@ } /* connect to server or proxy */ - if ((conn = http_connect(url, purl, flags)) == NULL) + if ((conn = http_connect(url, purl, flags, &cached)) == NULL) goto ouch; host = url->host; @@ -858,10 +896,10 @@ fetch_info("requesting %s://%s%s", url->scheme, host, url->doc); if (purl) { - http_cmd(conn, "%s %s://%s%s HTTP/1.1", + http_cmd(conn, "%s %s://%s%s HTTP/1.1\r\n", op, url->scheme, host, url->doc); } else { - http_cmd(conn, "%s %s HTTP/1.1", + http_cmd(conn, "%s %s HTTP/1.1\r\n", op, url->doc); } @@ -869,7 +907,7 @@ set_if_modified_since(conn, url->last_modified); /* virtual host */ - http_cmd(conn, "Host: %s", host); + http_cmd(conn, "Host: %s\r\n", host); /* proxy authorization */ if (purl) { @@ -897,19 +935,18 @@ /* other headers */ if ((p = getenv("HTTP_REFERER")) != NULL && *p != '\0') { if (strcasecmp(p, "auto") == 0) - http_cmd(conn, "Referer: %s://%s%s", + http_cmd(conn, "Referer: %s://%s%s\r\n", url->scheme, host, url->doc); else - http_cmd(conn, "Referer: %s", p); + http_cmd(conn, "Referer: %s\r\n", p); } if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0') - http_cmd(conn, "User-Agent: %s", p); + http_cmd(conn, "User-Agent: %s\r\n", p); else - http_cmd(conn, "User-Agent: %s ", _LIBFETCH_VER); + http_cmd(conn, "User-Agent: %s\r\n", _LIBFETCH_VER); if (url->offset > 0) - http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset); - http_cmd(conn, "Connection: close"); - http_cmd(conn, ""); + http_cmd(conn, "Range: bytes=%lld-\r\n", (long long)url->offset); + http_cmd(conn, "\r\n"); /* * Force the queued request to be dispatched. Normally, one @@ -973,6 +1010,9 @@ case HTTP_PROTOCOL_ERROR: /* fall through */ case -1: + --i; + if (cached) + continue; fetch_syserr(); goto ouch; default: @@ -991,6 +1031,10 @@ case hdr_error: http_seterr(HTTP_PROTOCOL_ERROR); goto ouch; + case hdr_connection: + /* XXX too weak? */ + keep_alive = (strcasecmp(p, "keep-alive") == 0); + break; case hdr_content_length: http_parse_length(p, &clength); break; @@ -1120,13 +1164,20 @@ URL->offset = offset; URL->length = clength; + if (clength == -1 && !chunked) + keep_alive = 0; + if (conn->err == HTTP_NOT_MODIFIED) { http_seterr(HTTP_NOT_MODIFIED); - return (NULL); + if (keep_alive) { + fetch_cache_put(conn, fetch_close); + conn = NULL; + } + goto ouch; } /* wrap it up in a fetchIO */ - if ((f = http_funopen(conn, chunked)) == NULL) { + if ((f = http_funopen(conn, chunked, keep_alive, clength)) == NULL) { fetch_syserr(); goto ouch; } @@ -1137,6 +1188,13 @@ fetchFreeURL(purl); if (HTTP_ERROR(conn->err)) { + + if (keep_alive) { + char buf[512]; + do { + } while (fetchIO_read(f, buf, sizeof(buf)) > 0); + } + fetchIO_close(f); f = NULL; } @@ -1223,7 +1281,7 @@ enum http_states state; }; -static size_t +static ssize_t parse_index(struct index_parser *parser, const char *buf, size_t len) { char *end_attr, p = *buf; @@ -1352,12 +1410,22 @@ return 0; *end_attr = '\0'; parser->state = ST_TAGA; - fetch_add_entry(parser->ue, parser->url, buf, 1); + if (fetch_add_entry(parser->ue, parser->url, buf, 1)) + return -1; return end_attr + 1 - buf; } + /* NOTREACHED */ abort(); } +struct http_index_cache { + struct http_index_cache *next; + struct url *location; + struct url_list ue; +}; + +static struct http_index_cache *index_cache; + /* * List a directory */ @@ -1366,17 +1434,53 @@ { fetchIO *f; char buf[2 * PATH_MAX]; - size_t buf_len, processed, sum_processed; - ssize_t read_len; + size_t buf_len, sum_processed; + ssize_t read_len, processed; struct index_parser state; + struct http_index_cache *cache = NULL; + int do_cache, ret; - state.url = url; - state.state = ST_NONE; - state.ue = ue; + do_cache = CHECK_FLAG('c'); + + if (do_cache) { + for (cache = index_cache; cache != NULL; cache = cache->next) { + if (strcmp(cache->location->scheme, url->scheme)) + continue; + if (strcmp(cache->location->user, url->user)) + continue; + if (strcmp(cache->location->pwd, url->pwd)) + continue; + if (strcmp(cache->location->host, url->host)) + continue; + if (cache->location->port != url->port) + continue; + if (strcmp(cache->location->doc, url->doc)) + continue; + return fetchAppendURLList(ue, &cache->ue); + } + + cache = malloc(sizeof(*cache)); + fetchInitURLList(&cache->ue); + cache->location = fetchCopyURL(url); + } f = fetchGetHTTP(url, flags); - if (f == NULL) + if (f == NULL) { + if (do_cache) { + fetchFreeURLList(&cache->ue); + fetchFreeURL(cache->location); + free(cache); + } return -1; + } + + state.url = url; + state.state = ST_NONE; + if (do_cache) { + state.ue = &cache->ue; + } else { + state.ue = ue; + } buf_len = 0; @@ -1385,12 +1489,31 @@ sum_processed = 0; do { processed = parse_index(&state, buf + sum_processed, buf_len); + if (processed == -1) + break; buf_len -= processed; sum_processed += processed; } while (processed != 0 && buf_len > 0); + if (processed == -1) { + read_len = -1; + break; + } memmove(buf, buf + sum_processed, buf_len); } fetchIO_close(f); - return read_len < 0 ? -1 : 0; + + ret = read_len < 0 ? -1 : 0; + + if (do_cache) { + if (ret == 0) { + cache->next = index_cache; + index_cache = cache; + } + + if (fetchAppendURLList(ue, &cache->ue)) + ret = -1; + } + + return ret; } Index: src/external/bsd/fetch/dist/libfetch/http.errors diff -u src/external/bsd/fetch/dist/libfetch/http.errors:1.1.1.2.8.1 src/external/bsd/fetch/dist/libfetch/http.errors:1.1.1.2.8.2 --- src/external/bsd/fetch/dist/libfetch/http.errors:1.1.1.2.8.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/http.errors Mon Feb 15 00:47:56 2010 @@ -1,5 +1,5 @@ # $FreeBSD: http.errors,v 1.5 2001/05/23 18:52:02 des Exp $ -# $NetBSD: http.errors,v 1.1.1.2.8.1 2009/05/30 15:58:17 snj Exp $ +# $NetBSD: http.errors,v 1.1.1.2.8.2 2010/02/15 00:47:56 snj Exp $ # # This list is taken from RFC 2068. # Index: src/external/bsd/fetch/dist/libfetch/fetch.3 diff -u src/external/bsd/fetch/dist/libfetch/fetch.3:1.1.1.3.6.1 src/external/bsd/fetch/dist/libfetch/fetch.3:1.1.1.3.6.2 --- src/external/bsd/fetch/dist/libfetch/fetch.3:1.1.1.3.6.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/fetch.3 Mon Feb 15 00:47:55 2010 @@ -1,5 +1,6 @@ .\"- .\" Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav +.\" Copyright (c) 2010 Joerg Sonnenberger <jo...@netbsd.org> .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -24,9 +25,9 @@ .\" SUCH DAMAGE. .\" .\" $FreeBSD: fetch.3,v 1.64 2007/12/18 11:03:26 des Exp $ -.\" $NetBSD: fetch.3,v 1.1.1.3.6.1 2009/05/30 15:58:17 snj Exp $ +.\" $NetBSD: fetch.3,v 1.1.1.3.6.2 2010/02/15 00:47:55 snj Exp $ .\" -.Dd February 4, 2009 +.Dd January 22, 2010 .Dt FETCH 3 .Os .Sh NAME @@ -64,6 +65,8 @@ .Nm fetchUnquotePath , .Nm fetchUnquoteFilename , .Nm fetchStringifyURL , +.Nm fetchConnectionCacheInit , +.Nm fetchConnectionCacheClose , .Nm fetch .Nd file transfer functions .Sh LIBRARY @@ -131,6 +134,8 @@ .Fn fetchListFTP "struct url_list *list" "struct url *u" "const char *flags" .Ft void .Fn fetchInitURLList "struct url_list *ul" +.Ft int +.Fn fetchAppendURLList "struct url_list *dst" "const struct url_list *src" .Ft void .Fn fetchFreeURLList "struct url_list *ul" .Ft char * @@ -139,6 +144,10 @@ .Fn fetchUnquoteFilename "struct url *u" .Ft char * .Fn fetchStringifyURL "const struct url *u" +.Ft void +.Fn fetchConnectionCacheInit "int global" "int per_host" +.Ft void +.Fn fetchConnectionCacheClose "void" .Sh DESCRIPTION These functions implement a high-level library for retrieving and uploading files using Uniform Resource Locators (URLs). @@ -281,6 +290,13 @@ .Fn fetchInitURLList and the entries be freed by calling .Fn fetchFreeURLList . +The function +.Fn fetchAppendURLList +can be used to append one URL lists to another. +If the +.Ql c +(cache result) flag is specified, the library is allowed to internally +cache the result. .Pp .Fn fetchStringifyURL returns the URL as string. @@ -298,6 +314,15 @@ .Fn free after use. .Pp +.Fn fetchConnectionCacheInit +enables the connection cache. +The first argument specifies the global limit on cached connections. +The second argument specifies the host limit. +Entries are considered to specify the same host, if the host name +from the URL is identical, indepent of the address or address family. +.Fn fetchConnectionCacheClose +flushed the connection cache and closes all cached connections. +.Pp .Fn fetchXGet , .Fn fetchGet , .Fn fetchPut , Index: src/external/bsd/fetch/dist/libfetch/ftp.c diff -u src/external/bsd/fetch/dist/libfetch/ftp.c:1.1.1.3.6.1 src/external/bsd/fetch/dist/libfetch/ftp.c:1.1.1.3.6.2 --- src/external/bsd/fetch/dist/libfetch/ftp.c:1.1.1.3.6.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/ftp.c Mon Feb 15 00:47:55 2010 @@ -1,7 +1,7 @@ -/* $NetBSD: ftp.c,v 1.1.1.3.6.1 2009/05/30 15:58:17 snj Exp $ */ +/* $NetBSD: ftp.c,v 1.1.1.3.6.2 2010/02/15 00:47:55 snj Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav - * Copyright (c) 2008, 2009 Joerg Sonnenberger <jo...@netbsd.org> + * Copyright (c) 2008, 2009, 2010 Joerg Sonnenberger <jo...@netbsd.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -120,9 +120,6 @@ #define FTP_SYNTAX_ERROR 500 #define FTP_PROTOCOL_ERROR 999 -static struct url cached_host; -static conn_t *cached_connection; - #define isftpreply(foo) \ (isdigit((unsigned char)foo[0]) && \ isdigit((unsigned char)foo[1]) && \ @@ -217,7 +214,7 @@ return (-1); } - r = fetch_putln(conn, msg, len); + r = fetch_write(conn, msg, len); free(msg); if (r == -1) { @@ -299,7 +296,7 @@ end = file + strlen(file); else if ((end = strrchr(file, '/')) == NULL) return (0); - if ((e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY || + if ((e = ftp_cmd(conn, "PWD\r\n")) != FTP_WORKING_DIRECTORY || (e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) { ftp_seterr(e); return (-1); @@ -316,8 +313,8 @@ break; if (pwd[i] == '\0' && (file[i - 1] == '/' || file[i] == '/')) break; - if ((e = ftp_cmd(conn, "CDUP")) != FTP_FILE_ACTION_OK || - (e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY || + if ((e = ftp_cmd(conn, "CDUP\r\n")) != FTP_FILE_ACTION_OK || + (e = ftp_cmd(conn, "PWD\r\n")) != FTP_WORKING_DIRECTORY || (e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) { ftp_seterr(e); return (-1); @@ -334,7 +331,7 @@ return (0); /* Change to the directory all in one chunk (e.g., foo/bar/baz). */ - e = ftp_cmd(conn, "CWD %.*s", (int)(end - beg), beg); + e = ftp_cmd(conn, "CWD %.*s\r\n", (int)(end - beg), beg); if (e == FTP_FILE_ACTION_OK) return (0); #endif /* FTP_COMBINE_CWDS */ @@ -345,7 +342,7 @@ ++beg, ++i; for (++i; file + i < end && file[i] != '/'; ++i) /* nothing */ ; - e = ftp_cmd(conn, "CWD %.*s", file + i - beg, beg); + e = ftp_cmd(conn, "CWD %.*s\r\n", file + i - beg, beg); if (e != FTP_FILE_ACTION_OK) { ftp_seterr(e); return (-1); @@ -371,7 +368,7 @@ default: return (FTP_PROTOCOL_ERROR); } - if ((e = ftp_cmd(conn, "MODE %c", mode)) != FTP_OK) { + if ((e = ftp_cmd(conn, "MODE %c\r\n", mode)) != FTP_OK) { if (mode == 'S') { /* * Stream mode is supposed to be the default - so @@ -407,7 +404,7 @@ default: return (FTP_PROTOCOL_ERROR); } - if ((e = ftp_cmd(conn, "TYPE %c", type)) != FTP_OK) + if ((e = ftp_cmd(conn, "TYPE %c\r\n", type)) != FTP_OK) return (e); return (FTP_OK); @@ -436,7 +433,7 @@ return (-1); } - e = ftp_cmd(conn, "SIZE %.*s", filenamelen, filename); + e = ftp_cmd(conn, "SIZE %.*s\r\n", filenamelen, filename); if (e != FTP_FILE_STATUS) { ftp_seterr(e); return (-1); @@ -453,7 +450,7 @@ if (us->size == 0) us->size = -1; - e = ftp_cmd(conn, "MDTM %.*s", filenamelen, filename); + e = ftp_cmd(conn, "MDTM %.*s\r\n", filenamelen, filename); if (e != FTP_FILE_STATUS) { ftp_seterr(e); return (-1); @@ -565,6 +562,13 @@ return (-1); } +static int +ftp_disconnect(conn_t *conn) +{ + ftp_cmd(conn, "QUIT\r\n"); + return fetch_close(conn); +} + static void ftp_closefn(void *v) { @@ -583,12 +587,10 @@ return; } fetch_close(io->dconn); - io->dir = -1; io->dconn = NULL; + io->dir = -1; r = ftp_chkerr(io->cconn); - if (io->cconn == cached_connection && io->cconn->ref == 1) - cached_connection = NULL; - fetch_close(io->cconn); + fetch_cache_put(io->cconn, ftp_disconnect); free(io); return; } @@ -620,9 +622,12 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_arg, int mode, off_t offset, const char *flags) { - struct sockaddr_storage sa; - struct sockaddr_in6 *sin6; - struct sockaddr_in *sin4; + union anonymous { + struct sockaddr_storage ss; + struct sockaddr sa; + struct sockaddr_in6 sin6; + struct sockaddr_in sin4; + } u; const char *bindaddr; const char *filename; int filenamelen, type; @@ -650,16 +655,16 @@ goto ouch; /* find our own address, bind, and listen */ - l = sizeof(sa); - if (getsockname(conn->sd, (struct sockaddr *)&sa, &l) == -1) + l = sizeof(u.ss); + if (getsockname(conn->sd, &u.sa, &l) == -1) goto sysouch; - if (sa.ss_family == AF_INET6) - unmappedaddr((struct sockaddr_in6 *)&sa, &l); + if (u.ss.ss_family == AF_INET6) + unmappedaddr(&u.sin6, &l); retry_mode: /* open data socket */ - if ((sd = socket(sa.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) { + if ((sd = socket(u.ss.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) { fetch_syserr(); return (NULL); } @@ -673,16 +678,16 @@ /* send PASV command */ if (verbose) fetch_info("setting passive mode"); - switch (sa.ss_family) { + switch (u.ss.ss_family) { case AF_INET: - if ((e = ftp_cmd(conn, "PASV")) != FTP_PASSIVE_MODE) + if ((e = ftp_cmd(conn, "PASV\r\n")) != FTP_PASSIVE_MODE) goto ouch; break; case AF_INET6: - if ((e = ftp_cmd(conn, "EPSV")) != FTP_EPASSIVE_MODE) { + if ((e = ftp_cmd(conn, "EPSV\r\n")) != FTP_EPASSIVE_MODE) { if (e == -1) goto ouch; - if ((e = ftp_cmd(conn, "LPSV")) != + if ((e = ftp_cmd(conn, "LPSV\r\n")) != FTP_LPASSIVE_MODE) goto ouch; } @@ -742,32 +747,30 @@ /* seek to required offset */ if (offset) - if (ftp_cmd(conn, "REST %lu", (unsigned long)offset) != FTP_FILE_OK) + if (ftp_cmd(conn, "REST %lu\r\n", (unsigned long)offset) != FTP_FILE_OK) goto sysouch; /* construct sockaddr for data socket */ - l = sizeof(sa); - if (getpeername(conn->sd, (struct sockaddr *)&sa, &l) == -1) + l = sizeof(u.ss); + if (getpeername(conn->sd, &u.sa, &l) == -1) goto sysouch; - if (sa.ss_family == AF_INET6) - unmappedaddr((struct sockaddr_in6 *)&sa, &l); - switch (sa.ss_family) { + if (u.ss.ss_family == AF_INET6) + unmappedaddr(&u.sin6, &l); + switch (u.ss.ss_family) { case AF_INET6: - sin6 = (struct sockaddr_in6 *)&sa; if (e == FTP_EPASSIVE_MODE) - sin6->sin6_port = htons(port); + u.sin6.sin6_port = htons(port); else { - memcpy(&sin6->sin6_addr, addr + 2, 16); - memcpy(&sin6->sin6_port, addr + 19, 2); + memcpy(&u.sin6.sin6_addr, addr + 2, 16); + memcpy(&u.sin6.sin6_port, addr + 19, 2); } break; case AF_INET: - sin4 = (struct sockaddr_in *)&sa; if (e == FTP_EPASSIVE_MODE) - sin4->sin_port = htons(port); + u.sin4.sin_port = htons(port); else { - memcpy(&sin4->sin_addr, addr, 4); - memcpy(&sin4->sin_port, addr + 4, 2); + memcpy(&u.sin4.sin_addr, addr, 4); + memcpy(&u.sin4.sin_port, addr + 4, 2); } break; default: @@ -780,18 +783,18 @@ fetch_info("opening data connection"); bindaddr = getenv("FETCH_BIND_ADDRESS"); if (bindaddr != NULL && *bindaddr != '\0' && - fetch_bind(sd, sa.ss_family, bindaddr) != 0) + fetch_bind(sd, u.ss.ss_family, bindaddr) != 0) goto sysouch; - if (connect(sd, (struct sockaddr *)&sa, l) == -1) + if (connect(sd, &u.sa, l) == -1) goto sysouch; /* make the server initiate the transfer */ if (verbose) fetch_info("initiating transfer"); if (op_arg) - e = ftp_cmd(conn, "%s%s%s", oper, *op_arg ? " " : "", op_arg); + e = ftp_cmd(conn, "%s%s%s\r\n", oper, *op_arg ? " " : "", op_arg); else - e = ftp_cmd(conn, "%s %.*s", oper, + e = ftp_cmd(conn, "%s %.*s\r\n", oper, filenamelen, filename); if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION) goto ouch; @@ -806,9 +809,9 @@ char *ap; char hname[INET6_ADDRSTRLEN]; - switch (sa.ss_family) { + switch (u.ss.ss_family) { case AF_INET6: - ((struct sockaddr_in6 *)&sa)->sin6_port = 0; + u.sin6.sin6_port = 0; #ifdef IPV6_PORTRANGE arg = low ? IPV6_PORTRANGE_DEFAULT : IPV6_PORTRANGE_HIGH; if (setsockopt(sd, IPPROTO_IPV6, IPV6_PORTRANGE, @@ -817,7 +820,7 @@ #endif break; case AF_INET: - ((struct sockaddr_in *)&sa)->sin_port = 0; + u.sin4.sin_port = 0; #ifdef IP_PORTRANGE arg = low ? IP_PORTRANGE_DEFAULT : IP_PORTRANGE_HIGH; if (setsockopt(sd, IPPROTO_IP, IP_PORTRANGE, @@ -828,20 +831,19 @@ } if (verbose) fetch_info("binding data socket"); - if (bind(sd, (struct sockaddr *)&sa, l) == -1) + if (bind(sd, &u.sa, l) == -1) goto sysouch; if (listen(sd, 1) == -1) goto sysouch; /* find what port we're on and tell the server */ - if (getsockname(sd, (struct sockaddr *)&sa, &l) == -1) + if (getsockname(sd, &u.sa, &l) == -1) goto sysouch; - switch (sa.ss_family) { + switch (u.ss.ss_family) { case AF_INET: - sin4 = (struct sockaddr_in *)&sa; - a = ntohl(sin4->sin_addr.s_addr); - p = ntohs(sin4->sin_port); - e = ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d", + a = ntohl(u.sin4.sin_addr.s_addr); + p = ntohs(u.sin4.sin_port); + e = ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d\r\n", (a >> 24) & 0xff, (a >> 16) & 0xff, (a >> 8) & 0xff, a & 0xff, (p >> 8) & 0xff, p & 0xff); @@ -849,28 +851,27 @@ case AF_INET6: #define UC(b) (((int)b)&0xff) e = -1; - sin6 = (struct sockaddr_in6 *)&sa; - sin6->sin6_scope_id = 0; - if (getnameinfo((struct sockaddr *)&sa, l, + u.sin6.sin6_scope_id = 0; + if (getnameinfo(&u.sa, l, hname, sizeof(hname), NULL, 0, NI_NUMERICHOST) == 0) { - e = ftp_cmd(conn, "EPRT |%d|%s|%d|", 2, hname, - htons(sin6->sin6_port)); + e = ftp_cmd(conn, "EPRT |%d|%s|%d|\r\n", 2, hname, + htons(u.sin6.sin6_port)); if (e == -1) goto ouch; } if (e != FTP_OK) { - ap = (char *)&sin6->sin6_addr; + ap = (char *)&u.sin6.sin6_addr; e = ftp_cmd(conn, - "LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d", + "LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d\r\n", 6, 16, UC(ap[0]), UC(ap[1]), UC(ap[2]), UC(ap[3]), UC(ap[4]), UC(ap[5]), UC(ap[6]), UC(ap[7]), UC(ap[8]), UC(ap[9]), UC(ap[10]), UC(ap[11]), UC(ap[12]), UC(ap[13]), UC(ap[14]), UC(ap[15]), 2, - (ntohs(sin6->sin6_port) >> 8) & 0xff, - ntohs(sin6->sin6_port) & 0xff); + (ntohs(u.sin6.sin6_port) >> 8) & 0xff, + ntohs(u.sin6.sin6_port) & 0xff); } break; default: @@ -882,16 +883,16 @@ /* seek to required offset */ if (offset) - if (ftp_cmd(conn, "REST %llu", (unsigned long long)offset) != FTP_FILE_OK) + if (ftp_cmd(conn, "REST %llu\r\n", (unsigned long long)offset) != FTP_FILE_OK) goto sysouch; /* make the server initiate the transfer */ if (verbose) fetch_info("initiating transfer"); if (op_arg) - e = ftp_cmd(conn, "%s%s%s", oper, *op_arg ? " " : "", op_arg); + e = ftp_cmd(conn, "%s%s%s\r\n", oper, *op_arg ? " " : "", op_arg); else - e = ftp_cmd(conn, "%s %.*s", oper, + e = ftp_cmd(conn, "%s %.*s\r\n", oper, filenamelen, filename); if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION) goto ouch; @@ -942,11 +943,11 @@ if (user == NULL || *user == '\0') user = FTP_ANONYMOUS_USER; if (purl && url->port == fetch_default_port(url->scheme)) - e = ftp_cmd(conn, "USER %...@%s", user, url->host); + e = ftp_cmd(conn, "USER %...@%s\r\n", user, url->host); else if (purl) - e = ftp_cmd(conn, "USER %...@%s@%d", user, url->host, url->port); + e = ftp_cmd(conn, "USER %...@%s@%d\r\n", user, url->host, url->port); else - e = ftp_cmd(conn, "USER %s", user); + e = ftp_cmd(conn, "USER %s\r\n", user); /* did the server request a password? */ if (e == FTP_NEED_PASSWORD) { @@ -965,7 +966,7 @@ pbuf[sizeof(pbuf) - 1] = '\0'; pwd = pbuf; } - e = ftp_cmd(conn, "PASS %s", pwd); + e = ftp_cmd(conn, "PASS %s\r\n", pwd); } return (e); @@ -998,10 +999,23 @@ /* check for proxy */ if (purl) { /* XXX proxy authentication! */ - conn = fetch_connect(purl->host, purl->port, af, verbose); + /* XXX connetion caching */ + if (!purl->port) + purl->port = fetch_default_port(purl->scheme); + + conn = fetch_connect(purl, af, verbose); } else { /* no proxy, go straight to target */ - conn = fetch_connect(url->host, url->port, af, verbose); + if (!url->port) + url->port = fetch_default_port(url->scheme); + + while ((conn = fetch_cache_get(url, af)) != NULL) { + e = ftp_cmd(conn, "NOOP\r\n"); + if (e == FTP_OK) + return conn; + fetch_close(conn); + } + conn = fetch_connect(url, af, verbose); purl = NULL; } @@ -1031,61 +1045,6 @@ } /* - * Disconnect from server - */ -static void -ftp_disconnect(conn_t *conn) -{ - (void)ftp_cmd(conn, "QUIT"); - if (conn == cached_connection && conn->ref == 1) - cached_connection = NULL; - fetch_close(conn); -} - -/* - * Check if we're already connected - */ -static int -ftp_isconnected(struct url *url) -{ - return (cached_connection - && (strcmp(url->host, cached_host.host) == 0) - && (strcmp(url->user, cached_host.user) == 0) - && (strcmp(url->pwd, cached_host.pwd) == 0) - && (url->port == cached_host.port)); -} - -/* - * Check the cache, reconnect if no luck - */ -static conn_t * -ftp_cached_connect(struct url *url, struct url *purl, const char *flags) -{ - conn_t *conn; - int e; - - /* set default port */ - if (!url->port) - url->port = fetch_default_port(url->scheme); - - /* try to use previously cached connection */ - if (ftp_isconnected(url)) { - e = ftp_cmd(cached_connection, "NOOP"); - if (e == FTP_OK || e == FTP_SYNTAX_ERROR) - return (fetch_ref(cached_connection)); - } - - /* connect to server */ - if ((conn = ftp_connect(url, purl, flags)) == NULL) - return (NULL); - if (cached_connection) - ftp_disconnect(cached_connection); - cached_connection = fetch_ref(conn); - memcpy(&cached_host, url, sizeof(*url)); - return (conn); -} - -/* * Check the proxy settings */ static struct url * @@ -1143,7 +1102,7 @@ } /* connect to server */ - conn = ftp_cached_connect(url, purl, flags); + conn = ftp_connect(url, purl, flags); if (purl) fetchFreeURL(purl); if (conn == NULL) @@ -1174,14 +1133,17 @@ if (if_modified_since && url->last_modified > 0 && url->last_modified >= us->mtime) { + free(path); fetchLastErrCode = FETCH_UNCHANGED; snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged"); return NULL; } /* just a stat */ - if (strcmp(op, "STAT") == 0) + if (strcmp(op, "STAT") == 0) { + free(path); return fetchIO_unopen(NULL, NULL, NULL, NULL); + } if (strcmp(op, "STOR") == 0 || strcmp(op, "APPE") == 0) oflag = O_WRONLY; else @@ -1246,6 +1208,7 @@ char buf[2 * PATH_MAX], *eol, *eos; ssize_t len; size_t cur_off; + int ret; /* XXX What about proxies? */ if (pattern == NULL || strcmp(pattern, "*") == 0) @@ -1255,6 +1218,8 @@ return -1; cur_off = 0; + ret = 0; + while ((len = fetchIO_read(f, buf + cur_off, sizeof(buf) - cur_off)) > 0) { cur_off += len; while ((eol = memchr(buf, '\n', cur_off)) != NULL) { @@ -1266,11 +1231,15 @@ else eos = eol; *eos = '\0'; - fetch_add_entry(ue, url, buf, 0); + ret = fetch_add_entry(ue, url, buf, 0); + if (ret) + break; cur_off -= eol - buf + 1; memmove(buf, eol + 1, cur_off); } } + if (ret) + break; } if (cur_off != 0 || len < 0) { /* Not RFC conform, bail out. */ @@ -1278,5 +1247,5 @@ return -1; } fetchIO_close(f); - return 0; + return ret; } Index: src/external/bsd/fetch/dist/libfetch/fetch.c diff -u src/external/bsd/fetch/dist/libfetch/fetch.c:1.1.1.3.4.1 src/external/bsd/fetch/dist/libfetch/fetch.c:1.1.1.3.4.2 --- src/external/bsd/fetch/dist/libfetch/fetch.c:1.1.1.3.4.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/fetch.c Mon Feb 15 00:47:55 2010 @@ -1,4 +1,4 @@ -/* $NetBSD: fetch.c,v 1.1.1.3.4.1 2009/05/30 15:58:17 snj Exp $ */ +/* $NetBSD: fetch.c,v 1.1.1.3.4.2 2010/02/15 00:47:55 snj Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * Copyright (c) 2008 Joerg Sonnenberger <jo...@netbsd.org> @@ -80,9 +80,7 @@ fetchIO * fetchXGet(struct url *URL, struct url_stat *us, const char *flags) { - int direct; - direct = CHECK_FLAG('d'); if (us != NULL) { us->size = -1; us->atime = us->mtime = 0; @@ -116,9 +114,7 @@ fetchIO * fetchPut(struct url *URL, const char *flags) { - int direct; - direct = CHECK_FLAG('d'); if (strcasecmp(URL->scheme, SCHEME_FILE) == 0) return (fetchPutFile(URL, flags)); else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) @@ -138,9 +134,7 @@ int fetchStat(struct url *URL, struct url_stat *us, const char *flags) { - int direct; - direct = CHECK_FLAG('d'); if (us != NULL) { us->size = -1; us->atime = us->mtime = 0; @@ -165,9 +159,7 @@ fetchList(struct url_list *ue, struct url *URL, const char *pattern, const char *flags) { - int direct; - direct = CHECK_FLAG('d'); if (strcasecmp(URL->scheme, SCHEME_FILE) == 0) return (fetchListFile(ue, URL, pattern, flags)); else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) Index: src/external/bsd/fetch/dist/libfetch/fetch.cat3 diff -u src/external/bsd/fetch/dist/libfetch/fetch.cat3:1.1.1.2.6.1 src/external/bsd/fetch/dist/libfetch/fetch.cat3:1.1.1.2.6.2 --- src/external/bsd/fetch/dist/libfetch/fetch.cat3:1.1.1.2.6.1 Sat May 30 15:58:17 2009 +++ src/external/bsd/fetch/dist/libfetch/fetch.cat3 Mon Feb 15 00:47:55 2010 @@ -11,7 +11,7 @@ ffeettcchhSSttrriinnggiiffyyUURRLL, ffeettcchh -- file transfer functions LLIIBBRRAARRYY - library ``libfetch'' + File Transfer Library for URLs (libfetch, -lfetch) SSYYNNOOPPSSIISS ##iinncclluuddee <<ssttddiioo..hh>> @@ -108,6 +108,9 @@ _v_o_i_d ffeettcchhIInniittUURRLLLLiisstt(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_u_l); + _i_n_t + ffeettcchhAAppppeennddUURRLLLLiisstt(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_d_s_t, _c_o_n_s_t _s_t_r_u_c_t _u_r_l___l_i_s_t _*_s_r_c); + _v_o_i_d ffeettcchhFFrreeeeUURRLLLLiisstt(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_u_l); @@ -129,7 +132,7 @@ Scheme Syntax detailed in RFC 1738. A regular expression which produces this syntax is: - <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)? + <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)? If the URL does not seem to begin with a scheme name, it is assumed to be a local path. Only absolute path names are accepted. @@ -204,7 +207,10 @@ }; The list should be initialized by calling ffeettcchhIInniittUURRLLLLiisstt() and the - entries be freed by calling ffeettcchhFFrreeeeUURRLLLLiisstt(). + entries be freed by calling ffeettcchhFFrreeeeUURRLLLLiisstt(). The functio + ffeettcchhAAppppeennddUURRLLLLiisstt() can be used to append one URL lists to another. If + the `c' (cache result) flag is specified, the library is allowed to + internally cache the result. ffeettcchhSSttrriinnggiiffyyUURRLL() returns the URL as string. ffeettcchhUUnnqquuootteePPaatthh() returns the path name part of the URL with any quoting undone. Query @@ -470,13 +476,13 @@ The ffeettcchh library first appeared in FreeBSD 3.0. AAUUTTHHOORRSS - The ffeettcchh library was mostly written by Dag-Erling Sm/orgrav + The ffeettcchh library was mostly written by Dag-Erling Smørgrav <d...@freebsd.org> with numerous suggestions from Jordan K. Hubbard <j...@freebsd.org>, Eugene Skepner <e...@qub.com> and other FreeBSD develop- ers. It replaces the older ffttppiioo library written by Poul-Henning Kamp <p...@freebsd.org> and Jordan K. Hubbard <j...@freebsd.org>. - This manual page was written by Dag-Erling Sm/orgrav <d...@freebsd.org>. + This manual page was written by Dag-Erling Smørgrav <d...@freebsd.org>. BBUUGGSS Some parts of the library are not yet implemented. The most notable