Module Name: src
Committed By: riz
Date: Sat Jun 12 17:58:13 UTC 2010
Modified Files:
src/external/bsd/fetch/dist/libfetch [netbsd-4]: common.c common.h
fetch.3 fetch.c fetch.cat3 fetch.h file.c ftp.c http.c
Log Message:
Pull up following revision(s) (requested by joerg in ticket #1384):
external/bsd/fetch/dist/libfetch/common.c 1.1.1.6-8
external/bsd/fetch/dist/libfetch/common.h 1.1.1.5-6
external/bsd/fetch/dist/libfetch/fetch.3 1.1.1.7-8
external/bsd/fetch/dist/libfetch/fetch.c 1.1.1.8
external/bsd/fetch/dist/libfetch/fetch.cat3 1.1.1.4-5
external/bsd/fetch/dist/libfetch/fetch.h 1.1.1.5-7
external/bsd/fetch/dist/libfetch/file.c 1.1.1.5-6
external/bsd/fetch/dist/libfetch/ftp.c 1.1.1.7-10
external/bsd/fetch/dist/libfetch/http.c 1.1.1.5-7
Import libfetch-2.25:
- address a number of lint warnings
- fix strict-alignment issues for GCC 4.4
- fix a bug in the line reading optimisation
- do not reuse a FTP connection if there is an active transfer on it
libfetch-2.26:
- Add support to aggressively cache directory listenings, useful for
HTTP
- Avoid leaking memory in error cases. From Xavier from Arch Linux.
libfetch-2.30:
- Revamped connection cache, allowing more than one active session
- HTTP keep-alive support
To generate a diff of this commit:
cvs rdiff -u -r1.1.1.5.4.2 -r1.1.1.5.4.3 \
src/external/bsd/fetch/dist/libfetch/common.c \
src/external/bsd/fetch/dist/libfetch/fetch.h \
src/external/bsd/fetch/dist/libfetch/file.c \
src/external/bsd/fetch/dist/libfetch/http.c
cvs rdiff -u -r1.1.1.4.4.2 -r1.1.1.4.4.3 \
src/external/bsd/fetch/dist/libfetch/common.h
cvs rdiff -u -r1.1.1.6.4.2 -r1.1.1.6.4.3 \
src/external/bsd/fetch/dist/libfetch/fetch.3 \
src/external/bsd/fetch/dist/libfetch/ftp.c
cvs rdiff -u -r1.1.1.7.4.2 -r1.1.1.7.4.3 \
src/external/bsd/fetch/dist/libfetch/fetch.c
cvs rdiff -u -r1.1.1.3.4.2 -r1.1.1.3.4.3 \
src/external/bsd/fetch/dist/libfetch/fetch.cat3
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/external/bsd/fetch/dist/libfetch/common.c
diff -u src/external/bsd/fetch/dist/libfetch/common.c:1.1.1.5.4.2 src/external/bsd/fetch/dist/libfetch/common.c:1.1.1.5.4.3
--- src/external/bsd/fetch/dist/libfetch/common.c:1.1.1.5.4.2 Fri Jun 5 17:19:28 2009
+++ src/external/bsd/fetch/dist/libfetch/common.c Sat Jun 12 17:58:12 2010
@@ -1,7 +1,7 @@
-/* $NetBSD: common.c,v 1.1.1.5.4.2 2009/06/05 17:19:28 snj Exp $ */
+/* $NetBSD: common.c,v 1.1.1.5.4.3 2010/06/12 17:58:12 riz Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
- * Copyright (c) 2008 Joerg Sonnenberger <[email protected]>
+ * Copyright (c) 2008, 2010 Joerg Sonnenberger <[email protected]>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -62,12 +62,13 @@
#include <string.h>
#include <unistd.h>
+#ifndef MSG_NOSIGNAL
+#include <signal.h>
+#endif
+
#include "fetch.h"
#include "common.h"
-#define DECONST(x,y) ((x)(uintptr_t)(y))
-
-
/*** Local data **************************************************************/
/*
@@ -83,10 +84,6 @@
{ -1, FETCH_UNKNOWN, "Unknown resolver error" }
};
-/* End-of-Line */
-static const char ENDL[2] = "\r\n";
-
-
/*** Error-reporting functions ***********************************************/
/*
@@ -234,22 +231,10 @@
/* allocate and fill connection structure */
if ((conn = calloc(1, sizeof(*conn))) == NULL)
return (NULL);
+ conn->cache_url = NULL;
conn->next_buf = NULL;
conn->next_len = 0;
conn->sd = sd;
- ++conn->ref;
- return (conn);
-}
-
-
-/*
- * Bump a connection's reference count.
- */
-conn_t *
-fetch_ref(conn_t *conn)
-{
-
- ++conn->ref;
return (conn);
}
@@ -261,17 +246,17 @@
fetch_bind(int sd, int af, const char *addr)
{
struct addrinfo hints, *res, *res0;
- int error;
memset(&hints, 0, sizeof(hints));
hints.ai_family = af;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = 0;
- if ((error = getaddrinfo(addr, NULL, &hints, &res0)) != 0)
+ if (getaddrinfo(addr, NULL, &hints, &res0))
return (-1);
- for (res = res0; res; res = res->ai_next)
+ for (res = res0; res; res = res->ai_next) {
if (bind(sd, res->ai_addr, res->ai_addrlen) == 0)
return (0);
+ }
return (-1);
}
@@ -280,7 +265,7 @@
* Establish a TCP connection to the specified port on the specified host.
*/
conn_t *
-fetch_connect(const char *host, int port, int af, int verbose)
+fetch_connect(struct url *url, int af, int verbose)
{
conn_t *conn;
char pbuf[10];
@@ -289,22 +274,22 @@
int sd, error;
if (verbose)
- fetch_info("looking up %s", host);
+ fetch_info("looking up %s", url->host);
/* look up host name and set up socket address structure */
- snprintf(pbuf, sizeof(pbuf), "%d", port);
+ snprintf(pbuf, sizeof(pbuf), "%d", url->port);
memset(&hints, 0, sizeof(hints));
hints.ai_family = af;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = 0;
- if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) {
+ if ((error = getaddrinfo(url->host, pbuf, &hints, &res0)) != 0) {
netdb_seterr(error);
return (NULL);
}
bindaddr = getenv("FETCH_BIND_ADDRESS");
if (verbose)
- fetch_info("connecting to %s:%d", host, port);
+ fetch_info("connecting to %s:%d", url->host, url->port);
/* try to connect */
for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {
@@ -331,9 +316,114 @@
fetch_syserr();
close(sd);
}
+ conn->cache_url = fetchCopyURL(url);
+ conn->cache_af = af;
return (conn);
}
+static conn_t *connection_cache;
+static int cache_global_limit = 0;
+static int cache_per_host_limit = 0;
+
+/*
+ * Initialise cache with the given limits.
+ */
+void
+fetchConnectionCacheInit(int global_limit, int per_host_limit)
+{
+
+ if (global_limit < 0)
+ cache_global_limit = INT_MAX;
+ else if (per_host_limit > global_limit)
+ cache_global_limit = per_host_limit;
+ else
+ cache_global_limit = global_limit;
+ if (per_host_limit < 0)
+ cache_per_host_limit = INT_MAX;
+ else
+ cache_per_host_limit = per_host_limit;
+}
+
+/*
+ * Flush cache and free all associated resources.
+ */
+void
+fetchConnectionCacheClose(void)
+{
+ conn_t *conn;
+
+ while ((conn = connection_cache) != NULL) {
+ connection_cache = conn->next_cached;
+ (*conn->cache_close)(conn);
+ }
+}
+
+/*
+ * Check connection cache for an existing entry matching
+ * protocol/host/port/user/password/family.
+ */
+conn_t *
+fetch_cache_get(const struct url *url, int af)
+{
+ conn_t *conn, *last_conn = NULL;
+
+ for (conn = connection_cache; conn; conn = conn->next_cached) {
+ if (conn->cache_url->port == url->port &&
+ strcmp(conn->cache_url->scheme, url->scheme) == 0 &&
+ strcmp(conn->cache_url->host, url->host) == 0 &&
+ strcmp(conn->cache_url->user, url->user) == 0 &&
+ strcmp(conn->cache_url->pwd, url->pwd) == 0 &&
+ (conn->cache_af == AF_UNSPEC || af == AF_UNSPEC ||
+ conn->cache_af == af)) {
+ if (last_conn != NULL)
+ last_conn->next_cached = conn->next_cached;
+ else
+ connection_cache = conn->next_cached;
+ return conn;
+ }
+ }
+
+ return NULL;
+}
+
+/*
+ * Put the connection back into the cache for reuse.
+ * If the connection is freed due to LRU or if the cache
+ * is explicitly closed, the given callback is called.
+ */
+void
+fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *))
+{
+ conn_t *iter, *last;
+ int global_count, host_count;
+
+ if (conn->cache_url == NULL || cache_global_limit == 0) {
+ (*closecb)(conn);
+ return;
+ }
+
+ global_count = host_count = 0;
+ last = NULL;
+ for (iter = connection_cache; iter;
+ last = iter, iter = iter->next_cached) {
+ ++global_count;
+ if (strcmp(conn->cache_url->host, iter->cache_url->host) == 0)
+ ++host_count;
+ if (global_count < cache_global_limit &&
+ host_count < cache_per_host_limit)
+ continue;
+ --global_count;
+ if (last != NULL)
+ last->next_cached = iter->next_cached;
+ else
+ connection_cache = iter->next_cached;
+ (*iter->cache_close)(iter);
+ }
+
+ conn->cache_close = closecb;
+ conn->next_cached = connection_cache;
+ connection_cache = conn;
+}
/*
* Enable SSL on a connection.
@@ -475,18 +565,22 @@
ssize_t len;
if (conn->buf == NULL) {
- if ((conn->buf = malloc(MIN_BUF_SIZE + 1)) == NULL) {
+ if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) {
errno = ENOMEM;
return (-1);
}
conn->bufsize = MIN_BUF_SIZE;
}
- conn->buf[0] = '\0';
conn->buflen = 0;
next = NULL;
do {
+ /*
+ * conn->bufsize != conn->buflen at this point,
+ * so the buffer can be NUL-terminated below for
+ * the case of len == 0.
+ */
len = fetch_read(conn, conn->buf + conn->buflen,
conn->bufsize - conn->buflen);
if (len == -1)
@@ -495,10 +589,13 @@
break;
next = memchr(conn->buf + conn->buflen, '\n', len);
conn->buflen += len;
- if (conn->buflen == conn->bufsize &&
- (next == NULL || next[1] == '\0')) {
+ if (conn->buflen == conn->bufsize && next == NULL) {
tmp = conn->buf;
- tmpsize = conn->bufsize * 2 + 1;
+ tmpsize = conn->bufsize * 2;
+ if (tmpsize < conn->bufsize) {
+ errno = ENOMEM;
+ return (-1);
+ }
if ((tmp = realloc(tmp, tmpsize)) == NULL) {
errno = ENOMEM;
return (-1);
@@ -509,39 +606,39 @@
} while (next == NULL);
if (next != NULL) {
+ *next = '\0';
conn->next_buf = next + 1;
conn->next_len = conn->buflen - (conn->next_buf - conn->buf);
conn->buflen = next - conn->buf;
+ } else {
+ conn->buf[conn->buflen] = '\0';
+ conn->next_len = 0;
}
- conn->buf[conn->buflen] = '\0';
return (0);
}
-
-/*
- * Write to a connection w/ timeout
- */
-ssize_t
-fetch_write(conn_t *conn, const char *buf, size_t len)
-{
- struct iovec iov;
-
- iov.iov_base = DECONST(char *, buf);
- iov.iov_len = len;
- return fetch_writev(conn, &iov, 1);
-}
-
/*
* Write a vector to a connection w/ timeout
* Note: can modify the iovec.
*/
ssize_t
-fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
+fetch_write(conn_t *conn, const void *buf, size_t len)
{
struct timeval now, timeout, waittv;
fd_set writefds;
ssize_t wlen, total;
int r;
+#ifndef MSG_NOSIGNAL
+ static int killed_sigpipe;
+#endif
+
+#ifndef MSG_NOSIGNAL
+ if (!killed_sigpipe) {
+ signal(SIGPIPE, SIG_IGN);
+ killed_sigpipe = 1;
+ }
+#endif
+
if (fetchTimeout) {
FD_ZERO(&writefds);
@@ -550,7 +647,7 @@
}
total = 0;
- while (iovcnt > 0) {
+ while (len) {
while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) {
FD_SET(conn->sd, &writefds);
gettimeofday(&now, NULL);
@@ -576,11 +673,14 @@
errno = 0;
#ifdef WITH_SSL
if (conn->ssl != NULL)
- wlen = SSL_write(conn->ssl,
- iov->iov_base, iov->iov_len);
+ wlen = SSL_write(conn->ssl, buf, len);
else
#endif
- wlen = writev(conn->sd, iov, iovcnt);
+#ifndef MSG_NOSIGNAL
+ wlen = send(conn->sd, buf, len, 0);
+#else
+ wlen = send(conn->sd, buf, len, MSG_NOSIGNAL);
+#endif
if (wlen == 0) {
/* we consider a short write a failure */
errno = EPIPE;
@@ -593,44 +693,14 @@
return (-1);
}
total += wlen;
- while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) {
- wlen -= iov->iov_len;
- iov++;
- iovcnt--;
- }
- if (iovcnt > 0) {
- iov->iov_len -= wlen;
- iov->iov_base = DECONST(char *, iov->iov_base) + wlen;
- }
+ buf = (const char *)buf + wlen;
+ len -= wlen;
}
return (total);
}
/*
- * Write a line of text to a connection w/ timeout
- */
-int
-fetch_putln(conn_t *conn, const char *str, size_t len)
-{
- struct iovec iov[2];
- int ret;
-
- iov[0].iov_base = DECONST(char *, str);
- iov[0].iov_len = len;
- iov[1].iov_base = DECONST(char *, ENDL);
- iov[1].iov_len = sizeof(ENDL);
- if (len == 0)
- ret = fetch_writev(conn, &iov[1], 1);
- else
- ret = fetch_writev(conn, iov, 2);
- if (ret == -1)
- return (-1);
- return (0);
-}
-
-
-/*
* Close connection
*/
int
@@ -638,9 +708,9 @@
{
int ret;
- if (--conn->ref > 0)
- return (0);
ret = close(conn->sd);
+ if (conn->cache_url)
+ fetchFreeURL(conn->cache_url);
free(conn->buf);
free(conn);
return (ret);
@@ -742,6 +812,40 @@
ue->urls = NULL;
}
+int
+fetchAppendURLList(struct url_list *dst, const struct url_list *src)
+{
+ size_t i, j, len;
+
+ len = dst->length + src->length;
+ if (len > dst->alloc_size) {
+ struct url *tmp;
+
+ tmp = realloc(dst->urls, len * sizeof(*tmp));
+ if (tmp == NULL) {
+ errno = ENOMEM;
+ fetch_syserr();
+ return (-1);
+ }
+ dst->alloc_size = len;
+ dst->urls = tmp;
+ }
+
+ for (i = 0, j = dst->length; i < src->length; ++i, ++j) {
+ dst->urls[j] = src->urls[i];
+ dst->urls[j].doc = strdup(src->urls[i].doc);
+ if (dst->urls[j].doc == NULL) {
+ while (i-- > 0)
+ free(dst->urls[j].doc);
+ fetch_syserr();
+ return -1;
+ }
+ }
+ dst->length = len;
+
+ return 0;
+}
+
void
fetchFreeURLList(struct url_list *ue)
{
Index: src/external/bsd/fetch/dist/libfetch/fetch.h
diff -u src/external/bsd/fetch/dist/libfetch/fetch.h:1.1.1.5.4.2 src/external/bsd/fetch/dist/libfetch/fetch.h:1.1.1.5.4.3
--- src/external/bsd/fetch/dist/libfetch/fetch.h:1.1.1.5.4.2 Fri Jun 5 17:19:28 2009
+++ src/external/bsd/fetch/dist/libfetch/fetch.h Sat Jun 12 17:58:12 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: fetch.h,v 1.1.1.5.4.2 2009/06/05 17:19:28 snj Exp $ */
+/* $NetBSD: fetch.h,v 1.1.1.5.4.3 2010/06/12 17:58:12 riz Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
* All rights reserved.
@@ -153,10 +153,15 @@
/* URL listening */
void fetchInitURLList(struct url_list *);
+int fetchAppendURLList(struct url_list *, const struct url_list *);
void fetchFreeURLList(struct url_list *);
char *fetchUnquotePath(struct url *);
char *fetchUnquoteFilename(struct url *);
+/* Connection caching */
+void fetchConnectionCacheInit(int, int);
+void fetchConnectionCacheClose(void);
+
/* Authentication */
typedef int (*auth_t)(struct url *);
extern auth_t fetchAuthMethod;
Index: src/external/bsd/fetch/dist/libfetch/file.c
diff -u src/external/bsd/fetch/dist/libfetch/file.c:1.1.1.5.4.2 src/external/bsd/fetch/dist/libfetch/file.c:1.1.1.5.4.3
--- src/external/bsd/fetch/dist/libfetch/file.c:1.1.1.5.4.2 Fri Jun 5 17:19:28 2009
+++ src/external/bsd/fetch/dist/libfetch/file.c Sat Jun 12 17:58:13 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: file.c,v 1.1.1.5.4.2 2009/06/05 17:19:28 snj Exp $ */
+/* $NetBSD: file.c,v 1.1.1.5.4.3 2010/06/12 17:58:13 riz Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
* Copyright (c) 2008, 2009 Joerg Sonnenberger <[email protected]>
@@ -234,6 +234,7 @@
char *path;
struct dirent *de;
DIR *dir;
+ int ret;
if ((path = fetchUnquotePath(u)) == NULL) {
fetch_syserr();
@@ -248,13 +249,17 @@
return -1;
}
+ ret = 0;
+
while ((de = readdir(dir)) != NULL) {
if (pattern && fnmatch(pattern, de->d_name, 0) != 0)
continue;
- fetch_add_entry(ue, u, de->d_name, 0);
+ ret = fetch_add_entry(ue, u, de->d_name, 0);
+ if (ret)
+ break;
}
closedir(dir);
- return 0;
+ return ret;
}
Index: src/external/bsd/fetch/dist/libfetch/http.c
diff -u src/external/bsd/fetch/dist/libfetch/http.c:1.1.1.5.4.2 src/external/bsd/fetch/dist/libfetch/http.c:1.1.1.5.4.3
--- src/external/bsd/fetch/dist/libfetch/http.c:1.1.1.5.4.2 Fri Jun 5 17:19:28 2009
+++ src/external/bsd/fetch/dist/libfetch/http.c Sat Jun 12 17:58:13 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: http.c,v 1.1.1.5.4.2 2009/06/05 17:19:28 snj Exp $ */
+/* $NetBSD: http.c,v 1.1.1.5.4.3 2010/06/12 17:58:13 riz Exp $ */
/*-
* Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav
* Copyright (c) 2003 Thomas Klausner <[email protected]>
@@ -142,6 +142,7 @@
{
conn_t *conn; /* connection */
int chunked; /* chunked mode */
+ int keep_alive; /* keep-alive mode */
char *buf; /* chunk buffer */
size_t bufsize; /* size of chunk buffer */
ssize_t buflen; /* amount of data currently in buffer */
@@ -149,6 +150,7 @@
int eof; /* end-of-file flag */
int error; /* error flag */
size_t chunksize; /* remaining size of current chunk */
+ off_t contentlength; /* remaining size of the content */
};
/*
@@ -211,6 +213,9 @@
if (io->eof)
return (0);
+ if (io->contentlength >= 0 && (off_t)len > io->contentlength)
+ len = io->contentlength;
+
if (io->chunked == 0) {
if (http_growbuf(io, len) == -1)
return (-1);
@@ -218,6 +223,8 @@
io->error = 1;
return (-1);
}
+ if (io->contentlength)
+ io->contentlength -= io->buflen;
io->bufpos = 0;
return (io->buflen);
}
@@ -229,6 +236,8 @@
return (-1);
case 0:
io->eof = 1;
+ if (fetch_getln(io->conn) == -1)
+ return (-1);
return (0);
}
}
@@ -242,6 +251,8 @@
return (-1);
}
io->chunksize -= io->buflen;
+ if (io->contentlength >= 0)
+ io->contentlength -= io->buflen;
if (io->chunksize == 0) {
char endl[2];
@@ -309,9 +320,23 @@
{
struct httpio *io = (struct httpio *)v;
- fetch_close(io->conn);
- if (io->buf)
- free(io->buf);
+ if (io->keep_alive) {
+ int val;
+
+ val = 0;
+ setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NODELAY, &val,
+ sizeof(val));
+ fetch_cache_put(io->conn, fetch_close);
+#ifdef TCP_NOPUSH
+ val = 1;
+ setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val,
+ sizeof(val));
+#endif
+ } else {
+ fetch_close(io->conn);
+ }
+
+ free(io->buf);
free(io);
}
@@ -319,7 +344,7 @@
* Wrap a file descriptor up
*/
static fetchIO *
-http_funopen(conn_t *conn, int chunked)
+http_funopen(conn_t *conn, int chunked, int keep_alive, off_t clength)
{
struct httpio *io;
fetchIO *f;
@@ -330,6 +355,8 @@
}
io->conn = conn;
io->chunked = chunked;
+ io->contentlength = clength;
+ io->keep_alive = keep_alive;
f = fetchIO_unopen(io, http_readfn, http_writefn, http_closefn);
if (f == NULL) {
fetch_syserr();
@@ -350,6 +377,7 @@
hdr_error = -1,
hdr_end = 0,
hdr_unknown = 1,
+ hdr_connection,
hdr_content_length,
hdr_content_range,
hdr_last_modified,
@@ -363,6 +391,7 @@
hdr_t num;
const char *name;
} hdr_names[] = {
+ { hdr_connection, "Connection" },
{ hdr_content_length, "Content-Length" },
{ hdr_content_range, "Content-Range" },
{ hdr_last_modified, "Last-Modified" },
@@ -393,7 +422,7 @@
return (-1);
}
- r = fetch_putln(conn, msg, len);
+ r = fetch_write(conn, msg, len);
free(msg);
if (r == -1) {
@@ -634,7 +663,7 @@
free(upw);
if (auth == NULL)
return (-1);
- r = http_cmd(conn, "%s: Basic %s", hdr, auth);
+ r = http_cmd(conn, "%s: Basic %s\r\n", hdr, auth);
free(auth);
return (r);
}
@@ -676,7 +705,7 @@
* Connect to the correct HTTP server or proxy.
*/
static conn_t *
-http_connect(struct url *URL, struct url *purl, const char *flags)
+http_connect(struct url *URL, struct url *purl, const char *flags, int *cached)
{
conn_t *conn;
int af, verbose;
@@ -684,6 +713,8 @@
int val;
#endif
+ *cached = 1;
+
#ifdef INET6
af = AF_UNSPEC;
#else
@@ -706,7 +737,12 @@
return (NULL);
}
- if ((conn = fetch_connect(URL->host, URL->port, af, verbose)) == NULL)
+ if ((conn = fetch_cache_get(URL, af)) != NULL) {
+ *cached = 1;
+ return (conn);
+ }
+
+ if ((conn = fetch_connect(URL, af, verbose)) == NULL)
/* fetch_connect() has already set an error code */
return (NULL);
if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 &&
@@ -764,7 +800,7 @@
snprintf(buf, sizeof(buf), "%.3s, %02d %.3s %4d %02d:%02d:%02d GMT",
weekdays + tm.tm_wday * 3, tm.tm_mday, months + tm.tm_mon * 3,
tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec);
- http_cmd(conn, "If-Modified-Since: %s", buf);
+ http_cmd(conn, "If-Modified-Since: %s\r\n", buf);
}
@@ -784,7 +820,8 @@
{
conn_t *conn;
struct url *url, *new;
- int chunked, direct, if_modified_since, need_auth, noredirect, verbose;
+ int chunked, direct, if_modified_since, need_auth, noredirect;
+ int keep_alive, verbose, cached;
int e, i, n, val;
off_t offset, clength, length, size;
time_t mtime;
@@ -797,6 +834,7 @@
noredirect = CHECK_FLAG('A');
verbose = CHECK_FLAG('v');
if_modified_since = CHECK_FLAG('i');
+ keep_alive = 0;
if (direct && purl) {
fetchFreeURL(purl);
@@ -834,7 +872,7 @@
}
/* connect to server or proxy */
- if ((conn = http_connect(url, purl, flags)) == NULL)
+ if ((conn = http_connect(url, purl, flags, &cached)) == NULL)
goto ouch;
host = url->host;
@@ -858,10 +896,10 @@
fetch_info("requesting %s://%s%s",
url->scheme, host, url->doc);
if (purl) {
- http_cmd(conn, "%s %s://%s%s HTTP/1.1",
+ http_cmd(conn, "%s %s://%s%s HTTP/1.1\r\n",
op, url->scheme, host, url->doc);
} else {
- http_cmd(conn, "%s %s HTTP/1.1",
+ http_cmd(conn, "%s %s HTTP/1.1\r\n",
op, url->doc);
}
@@ -869,7 +907,7 @@
set_if_modified_since(conn, url->last_modified);
/* virtual host */
- http_cmd(conn, "Host: %s", host);
+ http_cmd(conn, "Host: %s\r\n", host);
/* proxy authorization */
if (purl) {
@@ -897,19 +935,18 @@
/* other headers */
if ((p = getenv("HTTP_REFERER")) != NULL && *p != '\0') {
if (strcasecmp(p, "auto") == 0)
- http_cmd(conn, "Referer: %s://%s%s",
+ http_cmd(conn, "Referer: %s://%s%s\r\n",
url->scheme, host, url->doc);
else
- http_cmd(conn, "Referer: %s", p);
+ http_cmd(conn, "Referer: %s\r\n", p);
}
if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0')
- http_cmd(conn, "User-Agent: %s", p);
+ http_cmd(conn, "User-Agent: %s\r\n", p);
else
- http_cmd(conn, "User-Agent: %s ", _LIBFETCH_VER);
+ http_cmd(conn, "User-Agent: %s\r\n", _LIBFETCH_VER);
if (url->offset > 0)
- http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);
- http_cmd(conn, "Connection: close");
- http_cmd(conn, "");
+ http_cmd(conn, "Range: bytes=%lld-\r\n", (long long)url->offset);
+ http_cmd(conn, "\r\n");
/*
* Force the queued request to be dispatched. Normally, one
@@ -973,6 +1010,9 @@
case HTTP_PROTOCOL_ERROR:
/* fall through */
case -1:
+ --i;
+ if (cached)
+ continue;
fetch_syserr();
goto ouch;
default:
@@ -991,6 +1031,10 @@
case hdr_error:
http_seterr(HTTP_PROTOCOL_ERROR);
goto ouch;
+ case hdr_connection:
+ /* XXX too weak? */
+ keep_alive = (strcasecmp(p, "keep-alive") == 0);
+ break;
case hdr_content_length:
http_parse_length(p, &clength);
break;
@@ -1120,13 +1164,20 @@
URL->offset = offset;
URL->length = clength;
+ if (clength == -1 && !chunked)
+ keep_alive = 0;
+
if (conn->err == HTTP_NOT_MODIFIED) {
http_seterr(HTTP_NOT_MODIFIED);
- return (NULL);
+ if (keep_alive) {
+ fetch_cache_put(conn, fetch_close);
+ conn = NULL;
+ }
+ goto ouch;
}
/* wrap it up in a fetchIO */
- if ((f = http_funopen(conn, chunked)) == NULL) {
+ if ((f = http_funopen(conn, chunked, keep_alive, clength)) == NULL) {
fetch_syserr();
goto ouch;
}
@@ -1137,6 +1188,13 @@
fetchFreeURL(purl);
if (HTTP_ERROR(conn->err)) {
+
+ if (keep_alive) {
+ char buf[512];
+ do {
+ } while (fetchIO_read(f, buf, sizeof(buf)) > 0);
+ }
+
fetchIO_close(f);
f = NULL;
}
@@ -1223,7 +1281,7 @@
enum http_states state;
};
-static size_t
+static ssize_t
parse_index(struct index_parser *parser, const char *buf, size_t len)
{
char *end_attr, p = *buf;
@@ -1352,12 +1410,22 @@
return 0;
*end_attr = '\0';
parser->state = ST_TAGA;
- fetch_add_entry(parser->ue, parser->url, buf, 1);
+ if (fetch_add_entry(parser->ue, parser->url, buf, 1))
+ return -1;
return end_attr + 1 - buf;
}
+ /* NOTREACHED */
abort();
}
+struct http_index_cache {
+ struct http_index_cache *next;
+ struct url *location;
+ struct url_list ue;
+};
+
+static struct http_index_cache *index_cache;
+
/*
* List a directory
*/
@@ -1366,17 +1434,53 @@
{
fetchIO *f;
char buf[2 * PATH_MAX];
- size_t buf_len, processed, sum_processed;
- ssize_t read_len;
+ size_t buf_len, sum_processed;
+ ssize_t read_len, processed;
struct index_parser state;
+ struct http_index_cache *cache = NULL;
+ int do_cache, ret;
- state.url = url;
- state.state = ST_NONE;
- state.ue = ue;
+ do_cache = CHECK_FLAG('c');
+
+ if (do_cache) {
+ for (cache = index_cache; cache != NULL; cache = cache->next) {
+ if (strcmp(cache->location->scheme, url->scheme))
+ continue;
+ if (strcmp(cache->location->user, url->user))
+ continue;
+ if (strcmp(cache->location->pwd, url->pwd))
+ continue;
+ if (strcmp(cache->location->host, url->host))
+ continue;
+ if (cache->location->port != url->port)
+ continue;
+ if (strcmp(cache->location->doc, url->doc))
+ continue;
+ return fetchAppendURLList(ue, &cache->ue);
+ }
+
+ cache = malloc(sizeof(*cache));
+ fetchInitURLList(&cache->ue);
+ cache->location = fetchCopyURL(url);
+ }
f = fetchGetHTTP(url, flags);
- if (f == NULL)
+ if (f == NULL) {
+ if (do_cache) {
+ fetchFreeURLList(&cache->ue);
+ fetchFreeURL(cache->location);
+ free(cache);
+ }
return -1;
+ }
+
+ state.url = url;
+ state.state = ST_NONE;
+ if (do_cache) {
+ state.ue = &cache->ue;
+ } else {
+ state.ue = ue;
+ }
buf_len = 0;
@@ -1385,12 +1489,31 @@
sum_processed = 0;
do {
processed = parse_index(&state, buf + sum_processed, buf_len);
+ if (processed == -1)
+ break;
buf_len -= processed;
sum_processed += processed;
} while (processed != 0 && buf_len > 0);
+ if (processed == -1) {
+ read_len = -1;
+ break;
+ }
memmove(buf, buf + sum_processed, buf_len);
}
fetchIO_close(f);
- return read_len < 0 ? -1 : 0;
+
+ ret = read_len < 0 ? -1 : 0;
+
+ if (do_cache) {
+ if (ret == 0) {
+ cache->next = index_cache;
+ index_cache = cache;
+ }
+
+ if (fetchAppendURLList(ue, &cache->ue))
+ ret = -1;
+ }
+
+ return ret;
}
Index: src/external/bsd/fetch/dist/libfetch/common.h
diff -u src/external/bsd/fetch/dist/libfetch/common.h:1.1.1.4.4.2 src/external/bsd/fetch/dist/libfetch/common.h:1.1.1.4.4.3
--- src/external/bsd/fetch/dist/libfetch/common.h:1.1.1.4.4.2 Fri Jun 5 17:19:28 2009
+++ src/external/bsd/fetch/dist/libfetch/common.h Sat Jun 12 17:58:12 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: common.h,v 1.1.1.4.4.2 2009/06/05 17:19:28 snj Exp $ */
+/* $NetBSD: common.h,v 1.1.1.4.4.3 2010/06/12 17:58:12 riz Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
* All rights reserved.
@@ -53,6 +53,7 @@
/* Connection */
typedef struct fetchconn conn_t;
+
struct fetchconn {
int sd; /* socket descriptor */
char *buf; /* buffer */
@@ -71,7 +72,11 @@
const SSL_METHOD *ssl_meth; /* SSL method */
# endif
#endif
- int ref; /* reference count */
+
+ struct url *cache_url;
+ int cache_af;
+ int (*cache_close)(conn_t *);
+ conn_t *next_cached;
};
/* Structure used for error message lists */
@@ -81,24 +86,20 @@
const char *string;
};
-/* for fetch_writev */
-struct iovec;
-
void fetch_seterr(struct fetcherr *, int);
void fetch_syserr(void);
void fetch_info(const char *, ...);
int fetch_default_port(const char *);
int fetch_default_proxy_port(const char *);
int fetch_bind(int, int, const char *);
-conn_t *fetch_connect(const char *, int, int, int);
+conn_t *fetch_cache_get(const struct url *, int);
+void fetch_cache_put(conn_t *, int (*)(conn_t *));
+conn_t *fetch_connect(struct url *, int, int);
conn_t *fetch_reopen(int);
-conn_t *fetch_ref(conn_t *);
int fetch_ssl(conn_t *, int);
ssize_t fetch_read(conn_t *, char *, size_t);
int fetch_getln(conn_t *);
-ssize_t fetch_write(conn_t *, const char *, size_t);
-ssize_t fetch_writev(conn_t *, struct iovec *, int);
-int fetch_putln(conn_t *, const char *, size_t);
+ssize_t fetch_write(conn_t *, const void *, size_t);
int fetch_close(conn_t *);
int fetch_add_entry(struct url_list *, struct url *, const char *, int);
int fetch_netrc_auth(struct url *url);
Index: src/external/bsd/fetch/dist/libfetch/fetch.3
diff -u src/external/bsd/fetch/dist/libfetch/fetch.3:1.1.1.6.4.2 src/external/bsd/fetch/dist/libfetch/fetch.3:1.1.1.6.4.3
--- src/external/bsd/fetch/dist/libfetch/fetch.3:1.1.1.6.4.2 Fri Jun 5 17:19:28 2009
+++ src/external/bsd/fetch/dist/libfetch/fetch.3 Sat Jun 12 17:58:12 2010
@@ -1,5 +1,6 @@
.\"-
.\" Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+.\" Copyright (c) 2010 Joerg Sonnenberger <[email protected]>
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
@@ -24,9 +25,9 @@
.\" SUCH DAMAGE.
.\"
.\" $FreeBSD: fetch.3,v 1.64 2007/12/18 11:03:26 des Exp $
-.\" $NetBSD: fetch.3,v 1.1.1.6.4.2 2009/06/05 17:19:28 snj Exp $
+.\" $NetBSD: fetch.3,v 1.1.1.6.4.3 2010/06/12 17:58:12 riz Exp $
.\"
-.Dd February 4, 2009
+.Dd January 22, 2010
.Dt FETCH 3
.Os
.Sh NAME
@@ -64,6 +65,8 @@
.Nm fetchUnquotePath ,
.Nm fetchUnquoteFilename ,
.Nm fetchStringifyURL ,
+.Nm fetchConnectionCacheInit ,
+.Nm fetchConnectionCacheClose ,
.Nm fetch
.Nd file transfer functions
.Sh LIBRARY
@@ -131,6 +134,8 @@
.Fn fetchListFTP "struct url_list *list" "struct url *u" "const char *flags"
.Ft void
.Fn fetchInitURLList "struct url_list *ul"
+.Ft int
+.Fn fetchAppendURLList "struct url_list *dst" "const struct url_list *src"
.Ft void
.Fn fetchFreeURLList "struct url_list *ul"
.Ft char *
@@ -139,6 +144,10 @@
.Fn fetchUnquoteFilename "struct url *u"
.Ft char *
.Fn fetchStringifyURL "const struct url *u"
+.Ft void
+.Fn fetchConnectionCacheInit "int global" "int per_host"
+.Ft void
+.Fn fetchConnectionCacheClose "void"
.Sh DESCRIPTION
These functions implement a high-level library for retrieving and
uploading files using Uniform Resource Locators (URLs).
@@ -281,6 +290,13 @@
.Fn fetchInitURLList
and the entries be freed by calling
.Fn fetchFreeURLList .
+The function
+.Fn fetchAppendURLList
+can be used to append one URL lists to another.
+If the
+.Ql c
+(cache result) flag is specified, the library is allowed to internally
+cache the result.
.Pp
.Fn fetchStringifyURL
returns the URL as string.
@@ -298,6 +314,15 @@
.Fn free
after use.
.Pp
+.Fn fetchConnectionCacheInit
+enables the connection cache.
+The first argument specifies the global limit on cached connections.
+The second argument specifies the host limit.
+Entries are considered to specify the same host, if the host name
+from the URL is identical, indepent of the address or address family.
+.Fn fetchConnectionCacheClose
+flushed the connection cache and closes all cached connections.
+.Pp
.Fn fetchXGet ,
.Fn fetchGet ,
.Fn fetchPut ,
Index: src/external/bsd/fetch/dist/libfetch/ftp.c
diff -u src/external/bsd/fetch/dist/libfetch/ftp.c:1.1.1.6.4.2 src/external/bsd/fetch/dist/libfetch/ftp.c:1.1.1.6.4.3
--- src/external/bsd/fetch/dist/libfetch/ftp.c:1.1.1.6.4.2 Fri Jun 5 17:19:28 2009
+++ src/external/bsd/fetch/dist/libfetch/ftp.c Sat Jun 12 17:58:13 2010
@@ -1,7 +1,7 @@
-/* $NetBSD: ftp.c,v 1.1.1.6.4.2 2009/06/05 17:19:28 snj Exp $ */
+/* $NetBSD: ftp.c,v 1.1.1.6.4.3 2010/06/12 17:58:13 riz Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
- * Copyright (c) 2008, 2009 Joerg Sonnenberger <[email protected]>
+ * Copyright (c) 2008, 2009, 2010 Joerg Sonnenberger <[email protected]>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -120,9 +120,6 @@
#define FTP_SYNTAX_ERROR 500
#define FTP_PROTOCOL_ERROR 999
-static struct url cached_host;
-static conn_t *cached_connection;
-
#define isftpreply(foo) \
(isdigit((unsigned char)foo[0]) && \
isdigit((unsigned char)foo[1]) && \
@@ -217,7 +214,7 @@
return (-1);
}
- r = fetch_putln(conn, msg, len);
+ r = fetch_write(conn, msg, len);
free(msg);
if (r == -1) {
@@ -299,7 +296,7 @@
end = file + strlen(file);
else if ((end = strrchr(file, '/')) == NULL)
return (0);
- if ((e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY ||
+ if ((e = ftp_cmd(conn, "PWD\r\n")) != FTP_WORKING_DIRECTORY ||
(e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) {
ftp_seterr(e);
return (-1);
@@ -316,8 +313,8 @@
break;
if (pwd[i] == '\0' && (file[i - 1] == '/' || file[i] == '/'))
break;
- if ((e = ftp_cmd(conn, "CDUP")) != FTP_FILE_ACTION_OK ||
- (e = ftp_cmd(conn, "PWD")) != FTP_WORKING_DIRECTORY ||
+ if ((e = ftp_cmd(conn, "CDUP\r\n")) != FTP_FILE_ACTION_OK ||
+ (e = ftp_cmd(conn, "PWD\r\n")) != FTP_WORKING_DIRECTORY ||
(e = ftp_pwd(conn, pwd, sizeof(pwd))) != FTP_OK) {
ftp_seterr(e);
return (-1);
@@ -334,7 +331,7 @@
return (0);
/* Change to the directory all in one chunk (e.g., foo/bar/baz). */
- e = ftp_cmd(conn, "CWD %.*s", (int)(end - beg), beg);
+ e = ftp_cmd(conn, "CWD %.*s\r\n", (int)(end - beg), beg);
if (e == FTP_FILE_ACTION_OK)
return (0);
#endif /* FTP_COMBINE_CWDS */
@@ -345,7 +342,7 @@
++beg, ++i;
for (++i; file + i < end && file[i] != '/'; ++i)
/* nothing */ ;
- e = ftp_cmd(conn, "CWD %.*s", file + i - beg, beg);
+ e = ftp_cmd(conn, "CWD %.*s\r\n", file + i - beg, beg);
if (e != FTP_FILE_ACTION_OK) {
ftp_seterr(e);
return (-1);
@@ -371,7 +368,7 @@
default:
return (FTP_PROTOCOL_ERROR);
}
- if ((e = ftp_cmd(conn, "MODE %c", mode)) != FTP_OK) {
+ if ((e = ftp_cmd(conn, "MODE %c\r\n", mode)) != FTP_OK) {
if (mode == 'S') {
/*
* Stream mode is supposed to be the default - so
@@ -407,7 +404,7 @@
default:
return (FTP_PROTOCOL_ERROR);
}
- if ((e = ftp_cmd(conn, "TYPE %c", type)) != FTP_OK)
+ if ((e = ftp_cmd(conn, "TYPE %c\r\n", type)) != FTP_OK)
return (e);
return (FTP_OK);
@@ -436,7 +433,7 @@
return (-1);
}
- e = ftp_cmd(conn, "SIZE %.*s", filenamelen, filename);
+ e = ftp_cmd(conn, "SIZE %.*s\r\n", filenamelen, filename);
if (e != FTP_FILE_STATUS) {
ftp_seterr(e);
return (-1);
@@ -453,7 +450,7 @@
if (us->size == 0)
us->size = -1;
- e = ftp_cmd(conn, "MDTM %.*s", filenamelen, filename);
+ e = ftp_cmd(conn, "MDTM %.*s\r\n", filenamelen, filename);
if (e != FTP_FILE_STATUS) {
ftp_seterr(e);
return (-1);
@@ -565,6 +562,13 @@
return (-1);
}
+static int
+ftp_disconnect(conn_t *conn)
+{
+ ftp_cmd(conn, "QUIT\r\n");
+ return fetch_close(conn);
+}
+
static void
ftp_closefn(void *v)
{
@@ -583,12 +587,10 @@
return;
}
fetch_close(io->dconn);
- io->dir = -1;
io->dconn = NULL;
+ io->dir = -1;
r = ftp_chkerr(io->cconn);
- if (io->cconn == cached_connection && io->cconn->ref == 1)
- cached_connection = NULL;
- fetch_close(io->cconn);
+ fetch_cache_put(io->cconn, ftp_disconnect);
free(io);
return;
}
@@ -620,9 +622,12 @@
ftp_transfer(conn_t *conn, const char *oper, const char *file, const char *op_arg,
int mode, off_t offset, const char *flags)
{
- struct sockaddr_storage sa;
- struct sockaddr_in6 *sin6;
- struct sockaddr_in *sin4;
+ union anonymous {
+ struct sockaddr_storage ss;
+ struct sockaddr sa;
+ struct sockaddr_in6 sin6;
+ struct sockaddr_in sin4;
+ } u;
const char *bindaddr;
const char *filename;
int filenamelen, type;
@@ -650,16 +655,16 @@
goto ouch;
/* find our own address, bind, and listen */
- l = sizeof(sa);
- if (getsockname(conn->sd, (struct sockaddr *)&sa, &l) == -1)
+ l = sizeof(u.ss);
+ if (getsockname(conn->sd, &u.sa, &l) == -1)
goto sysouch;
- if (sa.ss_family == AF_INET6)
- unmappedaddr((struct sockaddr_in6 *)&sa, &l);
+ if (u.ss.ss_family == AF_INET6)
+ unmappedaddr(&u.sin6, &l);
retry_mode:
/* open data socket */
- if ((sd = socket(sa.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) {
+ if ((sd = socket(u.ss.ss_family, SOCK_STREAM, IPPROTO_TCP)) == -1) {
fetch_syserr();
return (NULL);
}
@@ -673,16 +678,16 @@
/* send PASV command */
if (verbose)
fetch_info("setting passive mode");
- switch (sa.ss_family) {
+ switch (u.ss.ss_family) {
case AF_INET:
- if ((e = ftp_cmd(conn, "PASV")) != FTP_PASSIVE_MODE)
+ if ((e = ftp_cmd(conn, "PASV\r\n")) != FTP_PASSIVE_MODE)
goto ouch;
break;
case AF_INET6:
- if ((e = ftp_cmd(conn, "EPSV")) != FTP_EPASSIVE_MODE) {
+ if ((e = ftp_cmd(conn, "EPSV\r\n")) != FTP_EPASSIVE_MODE) {
if (e == -1)
goto ouch;
- if ((e = ftp_cmd(conn, "LPSV")) !=
+ if ((e = ftp_cmd(conn, "LPSV\r\n")) !=
FTP_LPASSIVE_MODE)
goto ouch;
}
@@ -742,32 +747,30 @@
/* seek to required offset */
if (offset)
- if (ftp_cmd(conn, "REST %lu", (unsigned long)offset) != FTP_FILE_OK)
+ if (ftp_cmd(conn, "REST %lu\r\n", (unsigned long)offset) != FTP_FILE_OK)
goto sysouch;
/* construct sockaddr for data socket */
- l = sizeof(sa);
- if (getpeername(conn->sd, (struct sockaddr *)&sa, &l) == -1)
+ l = sizeof(u.ss);
+ if (getpeername(conn->sd, &u.sa, &l) == -1)
goto sysouch;
- if (sa.ss_family == AF_INET6)
- unmappedaddr((struct sockaddr_in6 *)&sa, &l);
- switch (sa.ss_family) {
+ if (u.ss.ss_family == AF_INET6)
+ unmappedaddr(&u.sin6, &l);
+ switch (u.ss.ss_family) {
case AF_INET6:
- sin6 = (struct sockaddr_in6 *)&sa;
if (e == FTP_EPASSIVE_MODE)
- sin6->sin6_port = htons(port);
+ u.sin6.sin6_port = htons(port);
else {
- memcpy(&sin6->sin6_addr, addr + 2, 16);
- memcpy(&sin6->sin6_port, addr + 19, 2);
+ memcpy(&u.sin6.sin6_addr, addr + 2, 16);
+ memcpy(&u.sin6.sin6_port, addr + 19, 2);
}
break;
case AF_INET:
- sin4 = (struct sockaddr_in *)&sa;
if (e == FTP_EPASSIVE_MODE)
- sin4->sin_port = htons(port);
+ u.sin4.sin_port = htons(port);
else {
- memcpy(&sin4->sin_addr, addr, 4);
- memcpy(&sin4->sin_port, addr + 4, 2);
+ memcpy(&u.sin4.sin_addr, addr, 4);
+ memcpy(&u.sin4.sin_port, addr + 4, 2);
}
break;
default:
@@ -780,18 +783,18 @@
fetch_info("opening data connection");
bindaddr = getenv("FETCH_BIND_ADDRESS");
if (bindaddr != NULL && *bindaddr != '\0' &&
- fetch_bind(sd, sa.ss_family, bindaddr) != 0)
+ fetch_bind(sd, u.ss.ss_family, bindaddr) != 0)
goto sysouch;
- if (connect(sd, (struct sockaddr *)&sa, l) == -1)
+ if (connect(sd, &u.sa, l) == -1)
goto sysouch;
/* make the server initiate the transfer */
if (verbose)
fetch_info("initiating transfer");
if (op_arg)
- e = ftp_cmd(conn, "%s%s%s", oper, *op_arg ? " " : "", op_arg);
+ e = ftp_cmd(conn, "%s%s%s\r\n", oper, *op_arg ? " " : "", op_arg);
else
- e = ftp_cmd(conn, "%s %.*s", oper,
+ e = ftp_cmd(conn, "%s %.*s\r\n", oper,
filenamelen, filename);
if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION)
goto ouch;
@@ -806,9 +809,9 @@
char *ap;
char hname[INET6_ADDRSTRLEN];
- switch (sa.ss_family) {
+ switch (u.ss.ss_family) {
case AF_INET6:
- ((struct sockaddr_in6 *)&sa)->sin6_port = 0;
+ u.sin6.sin6_port = 0;
#ifdef IPV6_PORTRANGE
arg = low ? IPV6_PORTRANGE_DEFAULT : IPV6_PORTRANGE_HIGH;
if (setsockopt(sd, IPPROTO_IPV6, IPV6_PORTRANGE,
@@ -817,7 +820,7 @@
#endif
break;
case AF_INET:
- ((struct sockaddr_in *)&sa)->sin_port = 0;
+ u.sin4.sin_port = 0;
#ifdef IP_PORTRANGE
arg = low ? IP_PORTRANGE_DEFAULT : IP_PORTRANGE_HIGH;
if (setsockopt(sd, IPPROTO_IP, IP_PORTRANGE,
@@ -828,20 +831,19 @@
}
if (verbose)
fetch_info("binding data socket");
- if (bind(sd, (struct sockaddr *)&sa, l) == -1)
+ if (bind(sd, &u.sa, l) == -1)
goto sysouch;
if (listen(sd, 1) == -1)
goto sysouch;
/* find what port we're on and tell the server */
- if (getsockname(sd, (struct sockaddr *)&sa, &l) == -1)
+ if (getsockname(sd, &u.sa, &l) == -1)
goto sysouch;
- switch (sa.ss_family) {
+ switch (u.ss.ss_family) {
case AF_INET:
- sin4 = (struct sockaddr_in *)&sa;
- a = ntohl(sin4->sin_addr.s_addr);
- p = ntohs(sin4->sin_port);
- e = ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d",
+ a = ntohl(u.sin4.sin_addr.s_addr);
+ p = ntohs(u.sin4.sin_port);
+ e = ftp_cmd(conn, "PORT %d,%d,%d,%d,%d,%d\r\n",
(a >> 24) & 0xff, (a >> 16) & 0xff,
(a >> 8) & 0xff, a & 0xff,
(p >> 8) & 0xff, p & 0xff);
@@ -849,28 +851,27 @@
case AF_INET6:
#define UC(b) (((int)b)&0xff)
e = -1;
- sin6 = (struct sockaddr_in6 *)&sa;
- sin6->sin6_scope_id = 0;
- if (getnameinfo((struct sockaddr *)&sa, l,
+ u.sin6.sin6_scope_id = 0;
+ if (getnameinfo(&u.sa, l,
hname, sizeof(hname),
NULL, 0, NI_NUMERICHOST) == 0) {
- e = ftp_cmd(conn, "EPRT |%d|%s|%d|", 2, hname,
- htons(sin6->sin6_port));
+ e = ftp_cmd(conn, "EPRT |%d|%s|%d|\r\n", 2, hname,
+ htons(u.sin6.sin6_port));
if (e == -1)
goto ouch;
}
if (e != FTP_OK) {
- ap = (char *)&sin6->sin6_addr;
+ ap = (char *)&u.sin6.sin6_addr;
e = ftp_cmd(conn,
- "LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d",
+ "LPRT %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d\r\n",
6, 16,
UC(ap[0]), UC(ap[1]), UC(ap[2]), UC(ap[3]),
UC(ap[4]), UC(ap[5]), UC(ap[6]), UC(ap[7]),
UC(ap[8]), UC(ap[9]), UC(ap[10]), UC(ap[11]),
UC(ap[12]), UC(ap[13]), UC(ap[14]), UC(ap[15]),
2,
- (ntohs(sin6->sin6_port) >> 8) & 0xff,
- ntohs(sin6->sin6_port) & 0xff);
+ (ntohs(u.sin6.sin6_port) >> 8) & 0xff,
+ ntohs(u.sin6.sin6_port) & 0xff);
}
break;
default:
@@ -882,16 +883,16 @@
/* seek to required offset */
if (offset)
- if (ftp_cmd(conn, "REST %llu", (unsigned long long)offset) != FTP_FILE_OK)
+ if (ftp_cmd(conn, "REST %llu\r\n", (unsigned long long)offset) != FTP_FILE_OK)
goto sysouch;
/* make the server initiate the transfer */
if (verbose)
fetch_info("initiating transfer");
if (op_arg)
- e = ftp_cmd(conn, "%s%s%s", oper, *op_arg ? " " : "", op_arg);
+ e = ftp_cmd(conn, "%s%s%s\r\n", oper, *op_arg ? " " : "", op_arg);
else
- e = ftp_cmd(conn, "%s %.*s", oper,
+ e = ftp_cmd(conn, "%s %.*s\r\n", oper,
filenamelen, filename);
if (e != FTP_CONNECTION_ALREADY_OPEN && e != FTP_OPEN_DATA_CONNECTION)
goto ouch;
@@ -942,11 +943,11 @@
if (user == NULL || *user == '\0')
user = FTP_ANONYMOUS_USER;
if (purl && url->port == fetch_default_port(url->scheme))
- e = ftp_cmd(conn, "USER %...@%s", user, url->host);
+ e = ftp_cmd(conn, "USER %...@%s\r\n", user, url->host);
else if (purl)
- e = ftp_cmd(conn, "USER %...@%s@%d", user, url->host, url->port);
+ e = ftp_cmd(conn, "USER %...@%s@%d\r\n", user, url->host, url->port);
else
- e = ftp_cmd(conn, "USER %s", user);
+ e = ftp_cmd(conn, "USER %s\r\n", user);
/* did the server request a password? */
if (e == FTP_NEED_PASSWORD) {
@@ -965,7 +966,7 @@
pbuf[sizeof(pbuf) - 1] = '\0';
pwd = pbuf;
}
- e = ftp_cmd(conn, "PASS %s", pwd);
+ e = ftp_cmd(conn, "PASS %s\r\n", pwd);
}
return (e);
@@ -998,10 +999,23 @@
/* check for proxy */
if (purl) {
/* XXX proxy authentication! */
- conn = fetch_connect(purl->host, purl->port, af, verbose);
+ /* XXX connetion caching */
+ if (!purl->port)
+ purl->port = fetch_default_port(purl->scheme);
+
+ conn = fetch_connect(purl, af, verbose);
} else {
/* no proxy, go straight to target */
- conn = fetch_connect(url->host, url->port, af, verbose);
+ if (!url->port)
+ url->port = fetch_default_port(url->scheme);
+
+ while ((conn = fetch_cache_get(url, af)) != NULL) {
+ e = ftp_cmd(conn, "NOOP\r\n");
+ if (e == FTP_OK)
+ return conn;
+ fetch_close(conn);
+ }
+ conn = fetch_connect(url, af, verbose);
purl = NULL;
}
@@ -1031,61 +1045,6 @@
}
/*
- * Disconnect from server
- */
-static void
-ftp_disconnect(conn_t *conn)
-{
- (void)ftp_cmd(conn, "QUIT");
- if (conn == cached_connection && conn->ref == 1)
- cached_connection = NULL;
- fetch_close(conn);
-}
-
-/*
- * Check if we're already connected
- */
-static int
-ftp_isconnected(struct url *url)
-{
- return (cached_connection
- && (strcmp(url->host, cached_host.host) == 0)
- && (strcmp(url->user, cached_host.user) == 0)
- && (strcmp(url->pwd, cached_host.pwd) == 0)
- && (url->port == cached_host.port));
-}
-
-/*
- * Check the cache, reconnect if no luck
- */
-static conn_t *
-ftp_cached_connect(struct url *url, struct url *purl, const char *flags)
-{
- conn_t *conn;
- int e;
-
- /* set default port */
- if (!url->port)
- url->port = fetch_default_port(url->scheme);
-
- /* try to use previously cached connection */
- if (ftp_isconnected(url)) {
- e = ftp_cmd(cached_connection, "NOOP");
- if (e == FTP_OK || e == FTP_SYNTAX_ERROR)
- return (fetch_ref(cached_connection));
- }
-
- /* connect to server */
- if ((conn = ftp_connect(url, purl, flags)) == NULL)
- return (NULL);
- if (cached_connection)
- ftp_disconnect(cached_connection);
- cached_connection = fetch_ref(conn);
- memcpy(&cached_host, url, sizeof(*url));
- return (conn);
-}
-
-/*
* Check the proxy settings
*/
static struct url *
@@ -1143,7 +1102,7 @@
}
/* connect to server */
- conn = ftp_cached_connect(url, purl, flags);
+ conn = ftp_connect(url, purl, flags);
if (purl)
fetchFreeURL(purl);
if (conn == NULL)
@@ -1174,14 +1133,17 @@
if (if_modified_since && url->last_modified > 0 &&
url->last_modified >= us->mtime) {
+ free(path);
fetchLastErrCode = FETCH_UNCHANGED;
snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged");
return NULL;
}
/* just a stat */
- if (strcmp(op, "STAT") == 0)
+ if (strcmp(op, "STAT") == 0) {
+ free(path);
return fetchIO_unopen(NULL, NULL, NULL, NULL);
+ }
if (strcmp(op, "STOR") == 0 || strcmp(op, "APPE") == 0)
oflag = O_WRONLY;
else
@@ -1246,6 +1208,7 @@
char buf[2 * PATH_MAX], *eol, *eos;
ssize_t len;
size_t cur_off;
+ int ret;
/* XXX What about proxies? */
if (pattern == NULL || strcmp(pattern, "*") == 0)
@@ -1255,6 +1218,8 @@
return -1;
cur_off = 0;
+ ret = 0;
+
while ((len = fetchIO_read(f, buf + cur_off, sizeof(buf) - cur_off)) > 0) {
cur_off += len;
while ((eol = memchr(buf, '\n', cur_off)) != NULL) {
@@ -1266,11 +1231,15 @@
else
eos = eol;
*eos = '\0';
- fetch_add_entry(ue, url, buf, 0);
+ ret = fetch_add_entry(ue, url, buf, 0);
+ if (ret)
+ break;
cur_off -= eol - buf + 1;
memmove(buf, eol + 1, cur_off);
}
}
+ if (ret)
+ break;
}
if (cur_off != 0 || len < 0) {
/* Not RFC conform, bail out. */
@@ -1278,5 +1247,5 @@
return -1;
}
fetchIO_close(f);
- return 0;
+ return ret;
}
Index: src/external/bsd/fetch/dist/libfetch/fetch.c
diff -u src/external/bsd/fetch/dist/libfetch/fetch.c:1.1.1.7.4.2 src/external/bsd/fetch/dist/libfetch/fetch.c:1.1.1.7.4.3
--- src/external/bsd/fetch/dist/libfetch/fetch.c:1.1.1.7.4.2 Fri Jun 5 17:19:28 2009
+++ src/external/bsd/fetch/dist/libfetch/fetch.c Sat Jun 12 17:58:12 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: fetch.c,v 1.1.1.7.4.2 2009/06/05 17:19:28 snj Exp $ */
+/* $NetBSD: fetch.c,v 1.1.1.7.4.3 2010/06/12 17:58:12 riz Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
* Copyright (c) 2008 Joerg Sonnenberger <[email protected]>
@@ -80,9 +80,7 @@
fetchIO *
fetchXGet(struct url *URL, struct url_stat *us, const char *flags)
{
- int direct;
- direct = CHECK_FLAG('d');
if (us != NULL) {
us->size = -1;
us->atime = us->mtime = 0;
@@ -116,9 +114,7 @@
fetchIO *
fetchPut(struct url *URL, const char *flags)
{
- int direct;
- direct = CHECK_FLAG('d');
if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
return (fetchPutFile(URL, flags));
else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
@@ -138,9 +134,7 @@
int
fetchStat(struct url *URL, struct url_stat *us, const char *flags)
{
- int direct;
- direct = CHECK_FLAG('d');
if (us != NULL) {
us->size = -1;
us->atime = us->mtime = 0;
@@ -165,9 +159,7 @@
fetchList(struct url_list *ue, struct url *URL, const char *pattern,
const char *flags)
{
- int direct;
- direct = CHECK_FLAG('d');
if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
return (fetchListFile(ue, URL, pattern, flags));
else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
Index: src/external/bsd/fetch/dist/libfetch/fetch.cat3
diff -u src/external/bsd/fetch/dist/libfetch/fetch.cat3:1.1.1.3.4.2 src/external/bsd/fetch/dist/libfetch/fetch.cat3:1.1.1.3.4.3
--- src/external/bsd/fetch/dist/libfetch/fetch.cat3:1.1.1.3.4.2 Fri Jun 5 17:19:28 2009
+++ src/external/bsd/fetch/dist/libfetch/fetch.cat3 Sat Jun 12 17:58:12 2010
@@ -11,7 +11,7 @@
ffeettcchhSSttrriinnggiiffyyUURRLL, ffeettcchh -- file transfer functions
LLIIBBRRAARRYY
- library ``libfetch''
+ File Transfer Library for URLs (libfetch, -lfetch)
SSYYNNOOPPSSIISS
##iinncclluuddee <<ssttddiioo..hh>>
@@ -108,6 +108,9 @@
_v_o_i_d
ffeettcchhIInniittUURRLLLLiisstt(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_u_l);
+ _i_n_t
+ ffeettcchhAAppppeennddUURRLLLLiisstt(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_d_s_t, _c_o_n_s_t _s_t_r_u_c_t _u_r_l___l_i_s_t _*_s_r_c);
+
_v_o_i_d
ffeettcchhFFrreeeeUURRLLLLiisstt(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_u_l);
@@ -129,7 +132,7 @@
Scheme Syntax detailed in RFC 1738. A regular expression which produces
this syntax is:
- <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
+ <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
If the URL does not seem to begin with a scheme name, it is assumed to be
a local path. Only absolute path names are accepted.
@@ -204,7 +207,10 @@
};
The list should be initialized by calling ffeettcchhIInniittUURRLLLLiisstt() and the
- entries be freed by calling ffeettcchhFFrreeeeUURRLLLLiisstt().
+ entries be freed by calling ffeettcchhFFrreeeeUURRLLLLiisstt(). The functio
+ ffeettcchhAAppppeennddUURRLLLLiisstt() can be used to append one URL lists to another. If
+ the `c' (cache result) flag is specified, the library is allowed to
+ internally cache the result.
ffeettcchhSSttrriinnggiiffyyUURRLL() returns the URL as string. ffeettcchhUUnnqquuootteePPaatthh()
returns the path name part of the URL with any quoting undone. Query
@@ -470,13 +476,13 @@
The ffeettcchh library first appeared in FreeBSD 3.0.
AAUUTTHHOORRSS
- The ffeettcchh library was mostly written by Dag-Erling Sm/orgrav
+ The ffeettcchh library was mostly written by Dag-Erling Smørgrav
<[email protected]> with numerous suggestions from Jordan K. Hubbard
<[email protected]>, Eugene Skepner <[email protected]> and other FreeBSD develop-
ers. It replaces the older ffttppiioo library written by Poul-Henning Kamp
<[email protected]> and Jordan K. Hubbard <[email protected]>.
- This manual page was written by Dag-Erling Sm/orgrav <[email protected]>.
+ This manual page was written by Dag-Erling Smørgrav <[email protected]>.
BBUUGGSS
Some parts of the library are not yet implemented. The most notable
