Hi,
I have an application that is (ab)using the pause/continue feature of
libcurl.
The application is having the CURLOPT_WRITEFUNCTION write the http data to a
small buffer, pausing the transfer when this buffer space is running low.
The
issue is that, on occasion, the transfer is not properly resumed after
calling
`curl_easy_pause(... CURLPAUSE_CONT)`. The buffer with the data that caused
the
CURL_WRITEFUNC_PAUSE return is always delivered, but no new data is
received.
The application is using the normal `curl_multi_perform()` to drive the
transfer.
I suspect that this failure to resume the transfer occurs when (or at least
has
something to do with) data is buffered in the SSL backend and select/poll
will
not return any events on the socket (either due to the server has
transmitted
all data for the request, or that the server can't send any more data due to
being limited by the TCP window).
Attached is a test program that hopefully demonstrates the phenomenon (at
least
it does for me ;-) Is there an issue here or have I missed something?
Regards,
Anders
#include <assert.h>
#include <stdio.h>
#include <stdbool.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <curl/curl.h>
#define VIRTUAL_BUFFER_SIZE 8192
#define CURL_RECVBUF_SIZE 1024
/*
* Global variables
*/
CURLM *curl_multi_handle;
CURL *curl;
bool paused;
size_t total_length;
size_t buffered;
size_t downloaded;
static size_t
http_header(void *buffer, size_t size, size_t nmemb, void *userp)
{
const char content_length[] = "content-length:";
size_t n = strlen(content_length);
if (strncasecmp((char *)buffer, content_length, n) == 0)
total_length = strtoul(buffer + n + 1, NULL, 10);
return nmemb;
}
static size_t
http_body(void *buffer, size_t size, size_t nmemb, void *userp)
{
if (buffered + nmemb > VIRTUAL_BUFFER_SIZE) {
/* Not enough space in the "buffer" => pause transfer */
fprintf(stderr, "Pause (buffered=%zu)\n", buffered);
paused = true;
return CURL_WRITEFUNC_PAUSE;
}
buffered += nmemb;
return nmemb;
}
#define MIN(a,b) ((a) < (b) ? (a) : (b))
int
main(int argc, char **argv)
{
const char *url = "https://ftp.sunet.se/mirror/archlinux/archive/floppy/20060104/boot.img";;
int request_size = 32 << 10;
struct CURLMsg *msg;
int running_handles;
int msgs_left;
char range[128];
if (argc > 1)
url = argv[1];
curl_global_init(CURL_GLOBAL_DEFAULT);
curl_multi_handle = curl_multi_init();
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, CURL_RECVBUF_SIZE);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
snprintf(range, sizeof range, "%d-%d", 0, request_size - 1);
curl_easy_setopt(curl, CURLOPT_RANGE, range);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, http_header);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, http_body);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
curl_multi_add_handle(curl_multi_handle, curl);
total_length = request_size;
curl_multi_perform(curl_multi_handle, &running_handles);
while (running_handles > 0) {
/* Resume if paused now that we have space */
if (paused && buffered < VIRTUAL_BUFFER_SIZE/2) {
fprintf(stderr, "Continue (buffered=%zu)\n", buffered);
curl_easy_pause(curl, CURLPAUSE_CONT);
paused = false;
}
curl_multi_perform(curl_multi_handle, &running_handles);
/* Consume some data from the "buffer" */
if (buffered > 0) {
size_t n = MIN(buffered, 1024);
buffered -= n;
downloaded += n;
}
usleep(10*1000);
{
static size_t last_buffered = ~0, last_downloaded = ~0;
if (last_buffered != buffered || last_downloaded != downloaded) {
printf("run=%d buffered=%zu progress=%zu\n", running_handles, buffered, downloaded);
last_buffered = buffered;
last_downloaded = downloaded;
}
}
}
msg = curl_multi_info_read(curl_multi_handle, &msgs_left);
fprintf(stderr, "Connection finished: %s\n", curl_easy_strerror(msg->data.result));
downloaded += buffered;
printf("Total downloaded: %zu\n", downloaded);
curl_multi_remove_handle(curl_multi_handle, curl);
curl_easy_cleanup(curl);
curl_multi_cleanup(curl_multi_handle);
return 0;
}
-------------------------------------------------------------------
Unsubscribe: https://cool.haxx.se/list/listinfo/curl-library
Etiquette: https://curl.haxx.se/mail/etiquette.html
