I've written a cURL plugin for nbdkit, which lets you turn any HTTP/FTP/SSH/etc source into a network block device:
https://github.com/libguestfs/nbdkit/tree/master/plugins/curl Currently it works well for read-only connections. I tried adding write support to the plugin (see attached patch), however I couldn't get that to work at all. Can a single cURL handle be "switched" between reading and uploading? And if so, how do you do that? Or should the plugin open two handles (one for reading, one for writing)? Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones Read my programming and virtualization blog: http://rwmj.wordpress.com virt-df lists disk usage of guests without needing to install any software inside the virtual machine. Supports Linux and Windows. http://people.redhat.com/~rjones/virt-df/
>From 5882ac09d1d95c2b9267407457672a4567598be3 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" <[email protected]> Date: Sat, 21 Jun 2014 22:10:18 +0100 Subject: [PATCH] CURL : ADD WRITE SUPPORT NOT WORKING --- plugins/curl/curl.c | 75 +++++++++++++++++++++++++++++++++++-- plugins/curl/nbdkit-curl-plugin.pod | 10 +++-- 2 files changed, 78 insertions(+), 7 deletions(-) diff --git a/plugins/curl/curl.c b/plugins/curl/curl.c index f922832..7b99c06 100644 --- a/plugins/curl/curl.c +++ b/plugins/curl/curl.c @@ -31,6 +31,14 @@ * SUCH DAMAGE. */ +/* NB: The terminology used by libcurl is confusing! + * + * WRITEFUNCTION / write_cb is used when reading from the remote server + * READFUNCTION / read_cb is used when writing to the remote server. + * + * We use the same terminology as libcurl here. + */ + #include <config.h> #include <stdio.h> @@ -145,6 +153,8 @@ struct curl_handle { char errbuf[CURL_ERROR_SIZE]; char *write_buf; uint32_t write_count; + const char *read_buf; + uint32_t read_count; }; /* Translate CURLcode to nbdkit_error. */ @@ -156,6 +166,7 @@ struct curl_handle { static size_t header_cb (void *ptr, size_t size, size_t nmemb, void *opaque); static size_t write_cb (char *ptr, size_t size, size_t nmemb, void *opaque); +static size_t read_cb (void *ptr, size_t size, size_t nmemb, void *opaque); /* Create the per-connection handle. */ static void * @@ -241,12 +252,15 @@ CURLOPT_PROXY nbdkit_debug ("accept range supported (for HTTP/HTTPS)"); } - /* Get set up for reading. */ + /* Get set up for reading and writing. */ curl_easy_setopt (h->c, CURLOPT_HEADERFUNCTION, NULL); curl_easy_setopt (h->c, CURLOPT_HEADERDATA, NULL); - curl_easy_setopt (h->c, CURLOPT_NOBODY, 0); /* No Body, not nobody! */ curl_easy_setopt (h->c, CURLOPT_WRITEFUNCTION, write_cb); curl_easy_setopt (h->c, CURLOPT_WRITEDATA, h); + if (!readonly) { + curl_easy_setopt (h->c, CURLOPT_READFUNCTION, read_cb); + curl_easy_setopt (h->c, CURLOPT_READDATA, h); + } nbdkit_debug ("returning new handle %p", h); @@ -307,7 +321,7 @@ curl_get_size (void *handle) return h->exportsize; } -/* Read data from the file. */ +/* Read data from the remote server. */ static int curl_pread (void *handle, void *buf, uint32_t count, uint64_t offset) { @@ -321,6 +335,9 @@ curl_pread (void *handle, void *buf, uint32_t count, uint64_t offset) h->write_buf = buf; h->write_count = count; + curl_easy_setopt (h->c, CURLOPT_NOBODY, 0); /* No Body, not nobody! */ + curl_easy_setopt (h->c, CURLOPT_HTTPGET, 1); + /* Make an HTTP range request. */ snprintf (range, sizeof range, "%" PRIu64 "-%" PRIu64, offset, offset + count); @@ -361,6 +378,57 @@ write_cb (char *ptr, size_t size, size_t nmemb, void *opaque) return orig_realsize; } +/* Write data to the remote server. */ +static int +curl_pwrite (void *handle, const void *buf, uint32_t count, uint64_t offset) +{ + struct curl_handle *h = handle; + CURLcode r; + char range[128]; + + /* Tell the read_cb where we want the data to be read from. read_cb + * will update this if the data comes in multiple sections. + */ + h->read_buf = buf; + h->read_count = count; + + curl_easy_setopt (h->c, CURLOPT_UPLOAD, 1); + + /* Make an HTTP range request. */ + snprintf (range, sizeof range, "%" PRIu64 "-%" PRIu64, + offset, offset + count); + curl_easy_setopt (h->c, CURLOPT_RANGE, range); + + /* The assumption here is that curl will look after timeouts. */ + while (h->read_count > 0) { + r = curl_easy_perform (h->c); + if (r != CURLE_OK) { + display_curl_error (h, r, "pwrite: curl_easy_perform"); + return -1; + } + } + + return 0; +} + +static size_t +read_cb (void *ptr, size_t size, size_t nmemb, void *opaque) +{ + struct curl_handle *h = opaque; + size_t realsize = size * nmemb; + + assert (h->read_buf); + if (realsize > h->read_count) + return CURL_READFUNC_ABORT; + + memcpy (ptr, h->read_buf, realsize); + + h->read_count -= realsize; + h->read_buf += realsize; + + return realsize; +} + static struct nbdkit_plugin plugin = { .name = "curl", .version = PACKAGE_VERSION, @@ -373,6 +441,7 @@ static struct nbdkit_plugin plugin = { .close = curl_close, .get_size = curl_get_size, .pread = curl_pread, + .pwrite = curl_pwrite, }; NBDKIT_REGISTER_PLUGIN(plugin) diff --git a/plugins/curl/nbdkit-curl-plugin.pod b/plugins/curl/nbdkit-curl-plugin.pod index 79659a5..97b6e7d 100644 --- a/plugins/curl/nbdkit-curl-plugin.pod +++ b/plugins/curl/nbdkit-curl-plugin.pod @@ -8,7 +8,7 @@ nbdkit-curl-plugin - nbdkit curl plugin (HTTP, FTP, SSH and other protocols) nbdkit -r curl url=http://example.com/disk.img - nbdkit -r curl url=sftp://[email protected]/~/disk.img + nbdkit curl url=sftp://[email protected]/~/disk.img =head1 DESCRIPTION @@ -20,7 +20,9 @@ how it was compiled, but most versions will handle HTTP, HTTPS, FTP, FTPS and SFTP (see: S<C<curl -V>>). For more information about libcurl, see L<http://curl.haxx.se>. -This plugin can B<only> handle B<readonly> connections. +B<Note:> This plugin supports writable connections. However for HTTP, +you may not want nbdkit to issue POST requests to the remote server. +To force nbdkit to use a readonly connection, pass the I<-r> flag. =head2 EXAMPLES @@ -34,12 +36,12 @@ control ports and protocols used to serve NBD see L<nbdkit(1)>). You can also access SSH servers. This uses the SFTP protocol which is built into most SSH servers: - nbdkit -r curl url=sftp://example.com/~/disk.img + nbdkit curl url=sftp://example.com/~/disk.img You may need to specify a username and/or a password. In this example the password is read from stdin: - nbdkit -r curl url=sftp://example.com/~/disk.img username=fred password=- + nbdkit curl url=sftp://example.com/~/disk.img username=fred password=- =head1 PARAMETERS -- 1.9.0
------------------------------------------------------------------- List admin: http://cool.haxx.se/list/listinfo/curl-library Etiquette: http://curl.haxx.se/mail/etiquette.html
