Hi friends,
Please find the source code and use examples of functions curl_fopen()
and curl_fdopen() in attachement.
These allow associating an URL or a curl handle with a standard C FILE
pointer and use the standard C IO API on it.
There is a built-in CRLF conversion feature and a no-wait mode.
I wrote them for fun and as a proof of concept, so there is no doc. Look
at the prototypes and examples.
I think it can be useful for the community or as a base of a
non-callback API for our project, but I don't really know what to do it
with now that it mostly works! May be a curl doc example?
You will find use examples embedded within the source: simple download,
mail sending, copy a handle to another, mixing protocols and even using
an handle as the data source for another one.
Please feel free to use it at your convenience.
Now, the main drawbacks:
- It uses the GNU-specific function fopencookie(): there is no support
for other environment yet. An equivalent facility exists in BSD
(funopen), but I did not find one for other systems.
- Chunked output MUST be enabled, as we don't know the write data size
in advance.
- There is no way to get the CURL handle from the file outside the
package itself.
- A FILE-attached CURL handle is not reusable outside this context:
closing the FILE deletes it.
- Only limited seek support is available (of course!): rewind and flush
to request termination.
- By the non-callback nature of the C IO API, one must know at each time
if input or output is expected.
- Because file:// URLs do not support pausing, they are open as normal C
files.
- There's a lot of buffer copying, probably adding overhead (I did not
measure perfomance).
- As it is built outside libcurl, it has no access to the internal
structures.
- Redirection of upload/post request is not possible. This probably is
also the case for multi-pass authentication.
- FILE/curl tuned buffer size is important: better not change it!
Have fun!
Patrick
/*
* curl_fopen(), curl_fdopen(): open a curl handle as a FILE pointer.
* Patrick Monnerat, Dec 2022.
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <curl/curl.h>
/* Wait timeout. */
#define WAIT_TIMEOUT_MS (60 * 60 * 1000) /* 1 hour in milliseconds. */
/* Cookie state flags. */
#define FLAG_READ (1 << 0) /* Opened for input. */
#define FLAG_WRITE (1 << 1) /* Opened for output. */
#define FLAG_BINARY (1 << 2) /* Transfered data is binary. */
#define FLAG_APPEND (1 << 3) /* FTP append data to remote file. */
#define FLAG_NOWAIT (1 << 4) /* Do not wait in curl. */
#define FLAG_CR (1 << 5) /* A CRLF is being processed. */
#define FLAG_PERFORM (1 << 6) /* Can call curl_multi_perform(). */
#define FLAG_WEOF (1 << 7) /* End of output. */
#define FLAG_DONE (1 << 8) /* Request completed. */
struct cookie {
CURL * easy; /* Request easy handle. */
CURLM * multi; /* Request multi handle. */
struct curl_slist *headers; /* Request headers. */
FILE * fp; /* The associated file structure. */
char * iptr; /* Caller's input buffer pointer. */
const char * iend; /* Input buffer end pointer. */
const char * optr; /* Caller's output data pointer. */
const char * oend; /* Output data end pointer. */
CURLcode result; /* Request completion status. */
unsigned short pausemask; /* Current request pause flags. */
unsigned short flags; /* Request state flags. */
char stdiobuf[CURL_MAX_WRITE_SIZE]; /* Standard IO buffer. */
};
static ssize_t cookie_read(void *cookie, char *buf, size_t size);
static ssize_t cookie_write(void *cookie, const char *buf, size_t size);
static int cookie_seek(void *cookie, off64_t *offset, int whence);
static int cookie_close(void *cookie);
static const cookie_io_functions_t io_funcs = {
cookie_read,
cookie_write,
cookie_seek,
cookie_close
};
static int seterror(struct cookie *p, int errcode)
{
errno = errcode;
p->fp->_flags |= _IO_ERR_SEEN;
return -1;
}
/*
* Perform a single curl step.
* Return 1 if still running, 0 if terminated, -1 if error.
*/
static int cookie_perform(struct cookie *p)
{
int nhandles;
int nqmsg;
CURLMcode result;
CURLMsg *m;
if(p->flags & FLAG_DONE)
return 0;
/* Perform one step. */
p->flags &= ~FLAG_PERFORM;
result = curl_multi_perform(p->multi, &nhandles);
switch(result) {
case CURLM_OK:
break;
case CURLM_CALL_MULTI_PERFORM:
p->flags |= FLAG_PERFORM;
break;
default: /* Error. */
return seterror(p, EIO);
}
if(nhandles)
return 1; /* Still running. */
/* Request complete. Get request result status code and check for error. */
m = curl_multi_info_read(p->multi, &nqmsg);
if(m) {
p->flags |= FLAG_DONE;
p->result = m->data.result;
}
if(p->result)
return seterror(p, EIO);
return 0; /* Request complete. */
}
/*
* Wait for a curl event.
* Return 0 if OK, else -1.
*/
static int cookie_wait(struct cookie *p)
{
CURLMcode result;
struct curl_waitfd fds;
result = curl_multi_wait(p->multi, &fds, 0, WAIT_TIMEOUT_MS, NULL);
if(result)
return seterror(p, EIO);
return 0;
}
/* Destroy cookie. */
static void destroy_cookie(struct cookie *p)
{
if(p->multi) {
if(p->easy)
curl_multi_remove_handle(p->multi, p->easy);
curl_multi_cleanup(p->multi);
}
if(p->easy)
curl_easy_cleanup(p->easy);
curl_slist_free_all(p->headers);
free(p);
}
/* Called by curl to read caller's data. */
static size_t cookie_read_callback(char *buffer, size_t size, size_t nitems,
void *userdata)
{
struct cookie *p = (struct cookie *) userdata;
size_t n = 0;
if((p->flags & (FLAG_WEOF | FLAG_WRITE)) != FLAG_WRITE)
return 0;
size *= nitems; /* Max byte count. */
if(p->optr >= p->oend || (p->pausemask & CURLPAUSE_SEND)) {
/* No input data available yet. Maybe there is still some more data to
come but this is unknown yet. Pause output. */
p->pausemask |= CURLPAUSE_SEND;
return CURL_READFUNC_PAUSE;
}
if(p->flags & FLAG_BINARY) {
/* Do not alter binary data, copy unchanged. */
n = p->oend - p->optr;
if(n > size)
n = size;
memcpy(buffer, p->optr, n);
p->optr += n;
}
else {
/* Newlines have to be mapped to CRLFs. */
while(n < size && p->optr < p->oend) {
char c = *p->optr;
if(!(p->flags & FLAG_CR) && c == '\n') {
c = '\r';
p->flags |= FLAG_CR; /* Mark CR has been inserted. */
}
else {
p->flags &= ~FLAG_CR; /* Ready for another CR insertion. */
p->optr++;
}
buffer[n++] = c;
}
}
return n;
}
/* Called by curl to write local data. */
static size_t cookie_write_callback(char *buffer, size_t size, size_t nitems,
void *userdata)
{
struct cookie *p = (struct cookie *) userdata;
size_t n = p->iend - p->iptr;
size *= nitems; /* Data byte count. */
if(!size || !(p->flags & FLAG_READ))
return size;
if(n < size)
p->pausemask |= CURLPAUSE_RECV;
if(p->pausemask & CURLPAUSE_RECV)
return CURL_WRITEFUNC_PAUSE;
if(p->flags & FLAG_BINARY) {
/* Copy binary data unchanged. */
n = size;
memcpy(p->iptr, buffer, n);
p->iptr += n;
}
else {
/* Copy mapping CRLFs to newlines. */
for(n = 0; n < size && p->iptr < p->iend;) {
char c = buffer[n++];
if(p->flags & FLAG_CR) { /* CR seen ? */
p->flags &= ~FLAG_CR;
/* If not CRLF, reinsert the deleted CR and reprocess same character. */
if(c != '\n') {
c = '\r';
n--;
}
}
else if(c == '\r') {
p->flags |= FLAG_CR;
continue;
}
*p->iptr++ = c;
}
}
return n;
}
/* Called from fread(). */
static ssize_t cookie_read(void *cookie, char *buf, size_t size)
{
struct cookie *p = (struct cookie *) cookie;
size_t n = 0;
int ret = 1;
if(p->flags & FLAG_DONE)
return 0;
/* Latch input buffer. */
p->iptr = buf;
p->iend = buf + size;
p->flags |= FLAG_WEOF; /* Output is over. */
if(p->pausemask) {
p->pausemask = 0;
curl_easy_pause(p->easy, p->pausemask);
}
/* Repeat stepping over curl until some data in input buffer. */
for(;;) {
ret = cookie_perform(p);
n = p->iptr - buf; /* Number of bytes read. */
if(ret <= 0 || n || (p->flags & FLAG_DONE))
break; /* Some data read, EOF or error. */
if(!(p->flags & FLAG_PERFORM)) {
/* Need to wait. Check if allowed. */
if(p->flags & FLAG_NOWAIT) {
ret = seterror(p, EWOULDBLOCK);
break;
}
cookie_wait(p); /* Wait for data from curl. */
}
}
/* Not ready to receive anymore. */
p->iend = p->iptr; /* Prevent getting anymore data. */
if(n || ret >= 0)
return n;
return -1;
}
/* Called from fwrite(). */
static ssize_t cookie_write(void *cookie, const char *buf, size_t size)
{
struct cookie *p = (struct cookie *) cookie;
size_t n = 0;
if((p->flags & (FLAG_WEOF | FLAG_WRITE | FLAG_DONE)) != FLAG_WRITE)
return seterror(p, EIO);
/* Latch output buffer. */
p->optr = buf;
p->oend = buf + size;
if(p->pausemask & CURLPAUSE_SEND) {
p->pausemask &= ~CURLPAUSE_SEND;
curl_easy_pause(p->easy, p->pausemask);
}
/* Loop stepping over curl to output buffered data. */
while(!ferror(p->fp)) {
cookie_perform(p);
if(p->optr >= p->oend || (p->flags & FLAG_DONE))
break; /* All data sent or request completed. */
if(!(p->flags & FLAG_PERFORM)) {
/* Need to wait. Check if allowed. */
if(p->flags & FLAG_NOWAIT) {
if(p->optr != buf) /* Some data sent? */
break;
return seterror(p, EWOULDBLOCK);
}
cookie_wait(p); /* Wait for a curl event. */
}
}
p->oend = p->optr; /* Do not send leftover. */
return p->optr - buf;
}
/* Called from fseek(). */
static int cookie_seek(void *cookie, off64_t *offset, int whence)
{
struct cookie *p = (struct cookie *) cookie;
int ret = 0;
if(*offset)
return seterror(p, EINVAL); /* No function supports a non 0 offset. */
switch(whence) {
case SEEK_SET: /* Rewind. */
/* Cancel and restart the request. */
curl_multi_remove_handle(p->multi, p->easy);
p->flags &= ~(FLAG_CR | FLAG_PERFORM | FLAG_WEOF | FLAG_DONE);
p->pausemask = 0;
p->iend = p->iptr;
p->oend = p->optr;
p->result = CURLE_OK;
curl_multi_add_handle(p->multi, p->easy);
break;
case SEEK_END: /* Terminate request discarding remaining input. */
p->flags |= FLAG_WEOF;
while(!(p->flags & FLAG_DONE)) {
p->iptr = p->stdiobuf; /* Not used at this time. */
p->iend = p->stdiobuf + sizeof(p->stdiobuf);
if(p->pausemask) {
p->pausemask = 0;
curl_easy_pause(p->easy, p->pausemask);
}
cookie_perform(p);
if(!(p->flags & FLAG_PERFORM)) {
if(p->flags & FLAG_NOWAIT) {
ret = seterror(p, EWOULDBLOCK);
break;
}
cookie_wait(p);
}
}
p->iend = p->iptr;
break;
default:
ret = seterror(p, EINVAL);
break;
}
return ret;
}
/*
* Called from fclose().
* Attempt to complete the request and close cookie.
* Always succeeds.
*/
static int cookie_close(void *cookie)
{
struct cookie *p = (struct cookie *) cookie;
/* Flush not needed: there's no pending write data and read data is
discarded. Aborts the request if still running. */
destroy_cookie(p);
return 0;
}
/* Associate a curl easy handle with a FILE pointer. Internal version. */
static FILE *cookie_fdopen(CURL *easy, const char *mode, struct cookie **cp)
{
struct cookie *p = NULL;
FILE *f = NULL;
char mymode[3];
if(cp)
*cp = NULL;
/* Check parameters. */
if(!easy || !mode) {
errno = EFAULT;
return NULL;
}
/* Create the cookie structure. */
p = (struct cookie *) calloc(1, sizeof(*p));
if(!p)
return NULL;
/* Decode requested mode. */
mymode[0] = *mode;
mymode[1] = mymode[2] = '\0';
switch(mode[0]) {
case 'a':
p->flags |= FLAG_WRITE | FLAG_APPEND;
break;
case 'r':
p->flags |= FLAG_READ;
break;
case 'w':
p->flags |= FLAG_WRITE;
break;
default:
destroy_cookie(p);
errno = EINVAL;
return NULL;
}
while(*++mode) {
switch(*mode) {
case 'b':
p->flags |= FLAG_BINARY;
break;
case '+':
p->flags |= FLAG_READ | FLAG_WRITE;
mymode[1] = '+';
break;
case 'i': /* No wait. */
p->flags |= FLAG_NOWAIT;
break;
default:
destroy_cookie(p);
errno = EINVAL;
return NULL;
}
}
/* Create the multi handle for our request. */
p->multi = curl_multi_init();
if(!p->multi) {
destroy_cookie(p);
errno = ENOMEM;
return NULL;
}
f = fopencookie(p, mymode, io_funcs);
if(!f) {
int sverrno = errno;
destroy_cookie(p);
errno = sverrno;
}
else {
p->fp = f;
/* Bind curl handle with cookie. */
p->easy = easy;
curl_multi_add_handle(p->multi, easy);
if(p->flags & FLAG_READ) {
curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, cookie_write_callback);
curl_easy_setopt(easy, CURLOPT_WRITEDATA, p);
}
if(p->flags & FLAG_WRITE) {
curl_easy_setopt(easy, CURLOPT_READFUNCTION, cookie_read_callback);
curl_easy_setopt(easy, CURLOPT_READDATA, p);
}
curl_multi_add_handle(p->multi, p->easy);
setbuffer(f, p->stdiobuf, sizeof(p->stdiobuf));
if(cp)
*cp = p;
}
return f;
}
/* Associate a curl easy handle with a FILE pointer. */
FILE *curl_fdopen(CURL *easy, const char *mode)
{
return cookie_fdopen(easy, mode, NULL);
}
/* Open an URL as a FILE pointer. */
FILE *curl_fopen(const char *url, const char *mode)
{
FILE *f = NULL;
char *part;
struct cookie *p;
CURL *easy;
CURLU *u;
/* Check parameters. */
if(!url || !mode) {
errno = EFAULT;
return NULL;
}
/* If this is a file URL, use a normal fopen'ed file. */
u = curl_url();
if(!u) {
errno = ENOMEM;
return NULL;
}
if(curl_url_set(u, CURLUPART_URL, url,
CURLU_DEFAULT_SCHEME | CURLU_PATH_AS_IS)) {
curl_url_cleanup(u);
errno = EINVAL;
return NULL;
}
if(curl_url_get(u, CURLUPART_SCHEME, &part, 0)) {
curl_url_cleanup(u);
errno = ENOMEM;
return NULL;
}
if(curl_strequal(part, "file")) {
free(part);
if(curl_url_get(u, CURLUPART_PATH, &part, CURLU_URLDECODE)) {
curl_url_cleanup(u);
errno = ENOMEM;
return NULL;
}
f = fopen(part, mode);
free(part);
return f;
}
free(part);
/* Build the curl handle for the request. */
easy = curl_easy_init();
if(easy) {
curl_easy_setopt(easy, CURLOPT_URL, url);
f = cookie_fdopen(easy, mode, &p);
if(!f)
curl_easy_cleanup(easy);
else if(p->flags & FLAG_WRITE) {
p->headers = curl_slist_append(p->headers, "Transfer-Encoding: chunked");
p->headers = curl_slist_append(p->headers, "Expect:");
curl_easy_setopt(easy, CURLOPT_HTTPHEADER, p->headers);
curl_easy_setopt(easy, CURLOPT_APPEND, p->flags & FLAG_APPEND? 1L: 0L);
if(p->flags & FLAG_READ)
curl_easy_setopt(easy, CURLOPT_POST, 1L);
else {
curl_easy_setopt(easy, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(easy, CURLOPT_NOBODY, 1L);
}
}
}
return f;
}
#ifndef NO_MAIN
/* Tune these for a real test. */
#define DOWNLOAD_FILE_URL "https://curl.se/libcurl/c/tinytest.c";
#define BIG_TEXT_URL \
"https://raw.githubusercontent.com/curl/curl/master/lib/http.c";
#define MTA_URL "smtp://localhost"
#define LDAP_URL "ldap://example.com";
#define MAIL_SENDER "Curl F. Open <curl_fo...@example.com>"
#define MAIL_RECIPIENT "Undisclosed list members <l...@example.com>"
#define WEB_SERVICE_URL "http://ws.example.com/service.php";
/* Simple curl_fopen() use: download a file and display it. */
void example1()
{
FILE *f = curl_fopen(DOWNLOAD_FILE_URL, "r");
int c;
puts("\n--- Example 1 ---");
while((c = getc(f)) != EOF)
putchar(c);
fclose(f);
}
static const char *mail_address(const char *addr)
{
if(addr || addr[strlen(addr) - 1] == '>') {
const char *a = strrchr(addr, '<');
if(a)
addr = a;
}
return addr;
}
/* Simple curl_fdopen() use: send a text e-mail. */
void example2()
{
CURL *easy = curl_easy_init();
struct curl_slist *recipients = NULL;
FILE *f;
int err;
puts("\n--- Example 2 ---");
curl_easy_setopt(easy, CURLOPT_URL, MTA_URL);
curl_easy_setopt(easy, CURLOPT_MAIL_FROM, mail_address(MAIL_SENDER));
recipients = curl_slist_append(recipients, mail_address(MAIL_RECIPIENT));
curl_easy_setopt(easy, CURLOPT_MAIL_RCPT, recipients);
curl_easy_setopt(easy, CURLOPT_UPLOAD, 1L);
f = curl_fdopen(easy, "w");
fputs("MIME-Version: 1.0\n", f);
fprintf(f, "From: %s\n", MAIL_SENDER);
fputs("Subject: This is a test message\n\n", f);
fprintf(f, "Greetings,\n\n%s\n", MAIL_SENDER);
fseek(f, 0L, SEEK_END); /* Terminate request. */
puts(ferror(f)? "Error: mail not sent": "Mail successfully sent");
fclose(f);
curl_slist_free_all(recipients);
}
/* Post data to a web service and display the response. */
void example3()
{
FILE *f = curl_fopen(WEB_SERVICE_URL, "w+");
int c;
puts("\n--- Example 3 ---");
fputs("ping data", f);
while((c = getc(f)) != EOF)
putchar(c);
fclose(f);
}
/* Send a remote file to a web service and display the service reply. */
void example4()
{
FILE *fi = curl_fopen(DOWNLOAD_FILE_URL, "r");
FILE *fo = curl_fopen(WEB_SERVICE_URL, "w+");
int c;
puts("\n--- Example 4 ---");
while((c = getc(fi)) != EOF)
putc(c, fo);
while((c = getc(fo)) != EOF)
putchar(c);
fclose(fi);
fclose(fo);
}
/* Use the output of a curl request as cascaded input to another. */
void example5()
{
FILE *fi = curl_fopen(LDAP_URL, "r");
CURL *easy = curl_easy_init();
struct curl_slist *sl = NULL;
FILE *fo;
int c;
FILE *f;
puts("\n--- Example 5 ---");
curl_easy_setopt(easy, CURLOPT_URL, WEB_SERVICE_URL);
curl_easy_setopt(easy, CURLOPT_POST, 1L);
sl = curl_slist_append(sl, "Transfer-Encoding: chunked");
sl = curl_slist_append(sl, "Expect:");
curl_easy_setopt(easy, CURLOPT_HTTPHEADER, sl);
curl_easy_setopt(easy, CURLOPT_READFUNCTION, fread);
curl_easy_setopt(easy, CURLOPT_READDATA, fi);
curl_easy_setopt(easy, CURLOPT_SEEKFUNCTION, fseek);
curl_easy_setopt(easy, CURLOPT_SEEKDATA, fi);
fo = curl_fdopen(easy, "r");
while((c = getc(fo)) != EOF)
putchar(c);
fclose(fi);
fclose(fo);
curl_slist_free_all(sl);
}
/* Rewind example: display some lines around first occurrence of a string. */
void example6()
{
FILE *f = curl_fopen(BIG_TEXT_URL, "r");
int targetline = 0;
int c;
int line;
char buf[100];
puts("\n--- Example 6 ---");
for(line = 1; fgets(buf, sizeof(buf), f); line++) {
if(strstr(buf, "http_output_bearer")) {
targetline = line;
break;
}
}
if(!targetline)
puts("String not found");
else {
fseek(f, 0L, SEEK_SET); /* Rewind. */
for(line = 1; fgets(buf, sizeof(buf), f); line++)
if(line > targetline - 8) {
printf("%5d %s", line, buf);
if(line > targetline + 8)
break;
}
}
fclose(f);
}
int main(int argc, char **argv)
{
curl_global_init(CURL_GLOBAL_DEFAULT);
example1();
example2();
example3();
example4();
example5();
example6();
curl_global_cleanup();
return 0;
}
#endif
--
Unsubscribe: https://lists.haxx.se/listinfo/curl-library
Etiquette: https://curl.se/mail/etiquette.html
