Daniel:
I am using libcurl and some of Curl Website example code as calling
routines for a multi-platform project where a webpage is read from
finance.yahoo.com.
...
Now when I perform the exact same access on Fedora 14 the read
performance is very slow. When the read access is executed by sending
the URL
I don't understand this part. The read access is executed?
there seems to be a pause of one or two minutes to get a response.
What!? Let me check I understand you correctly: You get an up to 120
seconds pause when you use libcurl on this Fedora that you don't see
if the same program run on Windows or Solaris?
120 seconds is an insanely long time!
Yes I counted 60 seconds of delay from url_fopen to url_fgets.
Code section is shown below and entire funcion is attached.
Also the libcurl version varies in all three environments. However
this morning I updated the Fedora 14 environment to the libcurl
newest version 7.32 and the performance did not improve.
From what version did you upgrade?
4.2.0
Do you only transfer plain HTTP?
Yes application requires only http.
Have you tried it on more than one specific Fedora host?
No Not so far.
Can you tell us which of the example codes that shows this problem if
you run it against the yahoo http server? Are you using the example
unmodified or can you provide us with the code you're trying?
Yes: See the entire source code file which is attached attached.
My read function is based on url_fopen.c
/******************************************************************************/
/* */
/* Read The Data From The Yahoo Web
Site: */
/* */
/******************************************************************************/
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Yahoo Web Site Read\n" );
handle = url_fopen ( URL_String, "r" );
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Handle = %d\n", handle );
if( !handle )
{
printf( "Couldn't Open URL: url_fopen() %s\n", url );
printf ( "Handle = %d\n", handle );
fclose( outf );
return -1;
}
/******************************************************************************/
/* */
/* Read The Price / Volume
Data: */
/* */
/******************************************************************************/
if ( Type == 'd' || Type == 'w' || Type == 'm' )
{ /* Start Type Price Data Block */
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Price Volume Read\n" );
curr_ptr = Stock_Data_Head;
prev_ptr = Stock_Data_Head;
prev_ptr->high = 1000000.0;
First_Record = TRUE;
Entry_Count = 0;
Price_High = 0.0;
Price_Low = 1000000000.0;
Split_Divisor = 1.0;
Divisor = 0.0;
Int_Divisor = 1.0;
while( !url_feof( handle ))
{ /* Start While */
url_fgets( buffer, sizeof( buffer ), handle);
if ( strstr ( buffer, "404 Not Found" ) != NULL )
{
printf( "Error 404 Symbol Not Found\n" );
fclose( outf );
return ( -1 );
}
if ( Print_Head->Print_Data == TRUE )
printf ( "%s", buffer );
if ( First_Record == FALSE )
{ /* Start Not First_Record */
sscanf ( buffer, "%4d", &year );
sscanf ( buffer+5, "%2d", &month );
sscanf ( buffer+8, "%2d", &day );
sscanf ( buffer+11, "%f,%f,%f,%f,%lli,%f", &open_read,
&high_read,
&low_read, &close_read, &volume_read, &adj_close_read );
/******************************************************************************/
/* */
/* TA_Bench read_yahoo.c 03/07/2013 */
/* */
/******************************************************************************/
/*****************************************************************************
*
* This example source code introduces a c library buffered I/O interface to
* URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
* rewind(). Supported functions have identical prototypes to their normal c
* lib namesakes and are preceaded by url_ .
*
* Using this code you can replace your program's fopen() with url_fopen()
* and fread() with url_fread() and it become possible to read remote streams
* instead of (only) local files. Local files (ie those that can be directly
* fopened) will drop back to using the underlying clib implementations
*
* See the main() function at the bottom that shows an app that retrives from a
* specified url using fgets() and fread() and saves as two output files.
*
* This example requires libcurl 7.9.7 or later.
*/
#include <stdio.h>
#include <string.h>
#ifndef WIN32
# include <sys/time.h>
#endif
#include <stdlib.h>
#include <errno.h>
#include <curl/curl.h>
/******************************************************************************/
/* */
/* Structures And Declarations. */
/* */
/******************************************************************************/
enum fcurl_type_e
{
CFTYPE_NONE=0,
CFTYPE_FILE=1,
CFTYPE_CURL=2
};
struct fcurl_data
{
enum fcurl_type_e type; /* type of handle */
union
{
CURL *curl;
FILE *file;
} handle; /* handle */
char *buffer; /* buffer to store cached data*/
size_t buffer_len; /* currently allocated buffers length */
size_t buffer_pos; /* end of data in buffer*/
int still_running; /* Is background url fetch still in progress */
};
typedef struct fcurl_data URL_FILE;
/* Exported Functions */
URL_FILE *url_fopen(const char *url,const char *operation);
int url_fclose(URL_FILE *file);
int url_feof(URL_FILE *file);
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
char * url_fgets(char *ptr, size_t size, URL_FILE *file);
void url_rewind(URL_FILE *file);
/* we use a global one for convenience */
CURLM *multi_handle;
// url="http://192.168.7.3/testfile";/* default to testurl */
// url="DineenConsulting.com";/* default to testurl */
// url="http://finance.yahoo.com/q/hp?s=SPY+Historical+Prices";;
/* default to testurl */
/******************************************************************************/
/* */
/* Function: Read_Yahoo_Primary */
/* */
/******************************************************************************/
int Read_Yahoo_Primary ( Stock_Data_Struct *Stock_Data_Head, Stock_Div_Struct
*Stock_Div_Head, Stock_Data_Info_Struct *Stock_Info_Head, Date_Struct
Date_Start, Date_Struct Date_End, char Symbol[10], char Type, Print_Struct
*Print_Head )
{
URL_FILE *handle;
FILE *outf;
FILE *Test_Handle;
int nread;
char buffer[256]; // Was 256
char Char_Temp[16];
const char *url;
char URL_String[256];
char Temp[256];
int year;
int month;
int day;
float open_read;
float high_read;
float low_read;
float close_read;
long long int volume_read;
float adj_close_read;
float open;
float high;
float low;
float close;
long long int volume;
float adj_close;
float dividend;
Stock_Data_Struct *curr_ptr;
Stock_Data_Struct *new_ptr;
Stock_Data_Struct *prev_ptr;
Stock_Div_Struct *div_curr_ptr;
Stock_Div_Struct *div_new_ptr;
Stock_Div_Struct *div_prev_ptr;
int First_Record;
int Entry_Count;
float Price_High;
float Price_Low;
float Split_Divisor;
float Divisor;
float Int_Divisor;
/******************************************************************************/
/* */
/* Print The Input Variables: */
/* */
/******************************************************************************/
if ( Print_Head->Print_Headers_Read == TRUE )
{
printf ( "Running Read_Yahoo\n") ;
printf ( "Start Date: Month = %d\n", Date_Start.Month );
printf ( "Start Date: Day = %d\n", Date_Start.Day );
printf ( "Start Date: Year = %d\n", Date_Start.Year );
printf ( "End Date: Month = %d\n", Date_End.Month );
printf ( "End Date: Day = %d\n", Date_End.Day );
printf ( "End Date: Year = %d\n", Date_End.Year );
printf ( "Symbol = %s\n", Symbol );
printf ( "Type = %c\n", Type );
}
/******************************************************************************/
/* */
/* Assemble The URL: */
/* */
/******************************************************************************/
strcpy ( URL_String, "http://ichart.finance.yahoo.com/table.csv?s="; );
strcat ( URL_String, Symbol );
strcat ( URL_String, "&a=" );
sprintf ( Temp, "%d", Date_Start.Month-1);
strcat ( URL_String, Temp );
strcat ( URL_String, "&b=" );
sprintf ( Temp, "%d", Date_Start.Day );
strcat ( URL_String, Temp );
strcat ( URL_String, "&c=" );
sprintf ( Temp, "%d", Date_Start.Year);
strcat ( URL_String, Temp );
strcat ( URL_String, "&g=" );
sprintf ( Temp, "%c", Type );
strcat ( URL_String, Temp );
strcat ( URL_String, "&d=" );
sprintf ( Temp, "%d", Date_End.Month-1);
strcat ( URL_String, Temp );
strcat ( URL_String, "&e=" );
sprintf ( Temp, "%d", Date_End.Day);
strcat ( URL_String, Temp );
strcat ( URL_String, "&f=" );
sprintf ( Temp, "%d", Date_End.Year);
strcat ( URL_String, Temp );
strcat ( URL_String, "&ignore=.csv" );
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "URL String = %s\n", URL_String );
/*
url="http://ichart.finance.yahoo.com/table.csv?s=SPY&d=10&e=2&f=2012&g=d&a=0&b=29&c=1993&ignore=.csv";;
*/
/*
url="http://ichart.finance.yahoo.com/table.csv?s=SPY&d=10&e=2&f=2012&g=d&a=0&b=29&c=1993&ignore=.csv";;
*/
/******************************************************************************/
/* */
/* Read The Data From The Yahoo Web Site: */
/* */
/******************************************************************************/
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Yahoo Web Site Read\n" );
handle = url_fopen ( URL_String, "r" );
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Handle = %d\n", handle );
if( !handle )
{
printf( "Couldn't Open URL: url_fopen() %s\n", url );
printf ( "Handle = %d\n", handle );
fclose( outf );
return -1;
}
/******************************************************************************/
/* */
/* Read The Price / Volume Data: */
/* */
/******************************************************************************/
if ( Type == 'd' || Type == 'w' || Type == 'm' )
{ /* Start Type Price Data Block */
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Price Volume Read\n" );
curr_ptr = Stock_Data_Head;
prev_ptr = Stock_Data_Head;
prev_ptr->high = 1000000.0;
First_Record = TRUE;
Entry_Count = 0;
Price_High = 0.0;
Price_Low = 1000000000.0;
Split_Divisor = 1.0;
Divisor = 0.0;
Int_Divisor = 1.0;
while( !url_feof( handle ))
{ /* Start While */
url_fgets( buffer, sizeof( buffer ), handle);
if ( strstr ( buffer, "404 Not Found" ) != NULL )
{
printf( "Error 404 Symbol Not Found\n" );
fclose( outf );
return ( -1 );
}
if ( Print_Head->Print_Data == TRUE )
printf ( "%s", buffer );
if ( First_Record == FALSE )
{ /* Start Not First_Record */
sscanf ( buffer, "%4d", &year );
sscanf ( buffer+5, "%2d", &month );
sscanf ( buffer+8, "%2d", &day );
sscanf ( buffer+11, "%f,%f,%f,%f,%lli,%f", &open_read, &high_read,
&low_read, &close_read, &volume_read, &adj_close_read );
if ( ( 1.40 * Split_Divisor * prev_ptr->high ) <= high_read )
{ /* Start Block Split Occured */
Divisor = ( high_read / (Split_Divisor*prev_ptr->high));
if ( Divisor >= 7.8 )
Int_Divisor = 8;
else
if ( Divisor >= 6.8 )
Int_Divisor = 7;
else
if ( Divisor >= 5.8 )
Int_Divisor = 6;
else
if ( Divisor >= 4.8 )
Int_Divisor = 5;
else
if ( Divisor >= 3.8 )
Int_Divisor = 4;
else
if ( Divisor >= 2.8 )
Int_Divisor = 3;
else
if ( Divisor >= 1.6 )
Int_Divisor = 2;
else
if ( Divisor >= 1.3 )
Int_Divisor = 1.5;
Split_Divisor = (float) Split_Divisor * Int_Divisor;
curr_ptr->Split_Occured = TRUE;
if ( Print_Head->Print_Data == TRUE )
printf ( "Split Detected: Date: %d/%d/%d Int_Divisor %d
Split_Divisor %f\n", month, day, year, Int_Divisor, Split_Divisor);
} /* End Block Split Occured */
else
{ /* Start Block No Split Occured */
Int_Divisor = 0.0;
curr_ptr->Split_Occured = FALSE;
}; /* End Block No Split Occured */
open = open_read / Split_Divisor;
high = high_read / Split_Divisor;
low = low_read / Split_Divisor;
close = close_read / Split_Divisor;
volume = volume_read;
adj_close = adj_close_read / Split_Divisor;
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Curr_Ptr %x\n", curr_ptr );
printf ( "Date: %4i %2i %2i\n", year, month, day );
printf ( "Open: %2.2f\n", open);
printf ( "High: %2.2f\n", high );
printf ( "Low: %2.2f\n", low );
printf ( "Close: %2.2f\n", close );
printf ( "Volume: %i\n", volume );
printf ( "Adj_Close: %2.2f\n", adj_close );
printf ( "High_Read: %2.2f\n", high_read);
printf ( "Prev High: %2.2f\n", prev_ptr->high );
printf ( "Divisor: %2.2f\n", Divisor );
printf ( "Split_Divisor: %2.2f\n", Split_Divisor );
printf ( "Int_Divisor: %2.2f\n\n", Int_Divisor );
};
curr_ptr->year = year;
curr_ptr->month = month;
curr_ptr->day = day;
curr_ptr->open = open;
curr_ptr->high = high;
curr_ptr->low = low;
curr_ptr->close = close;
curr_ptr->volume = volume;
curr_ptr->adj_close = adj_close;
curr_ptr->open_read = open_read;
curr_ptr->high_read = high_read;
curr_ptr->low_read = low_read;
curr_ptr->close_read = close_read;
curr_ptr->volume_read = volume_read;
curr_ptr->adj_close_read = adj_close_read;
curr_ptr->Int_Divisor = Int_Divisor;
curr_ptr->Split_Divisor = Split_Divisor;
curr_ptr->Entry_Count = Entry_Count;
if ( high > Price_High )
Price_High = high;
if ( low < Price_Low )
Price_Low = low;
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Price_High = %f\n", Price_High );
printf ( "Price_Low = %f\n", Price_Low );
};
new_ptr = malloc( sizeof( Stock_Data_Struct) );
curr_ptr->next_ptr = new_ptr;
curr_ptr->prev_ptr = prev_ptr;
prev_ptr = curr_ptr;
curr_ptr = new_ptr;
Entry_Count++;
}; /* End Not First_Record */
First_Record = FALSE;
}; /* End While */
if ( Entry_Count != NULL )
curr_ptr = prev_ptr;
curr_ptr->next_ptr = NULL;
Stock_Info_Head->Date_Start.Day = Date_Start.Day;
Stock_Info_Head->Date_Start.Month = Date_Start.Month;
Stock_Info_Head->Date_Start.Year= Date_Start.Year;
Stock_Info_Head->Date_End.Day = Date_End.Day;
Stock_Info_Head->Date_End.Month = Date_End.Month;
Stock_Info_Head->Date_End.Year = Date_End.Year;
strcpy ( Stock_Info_Head->Symbol, Symbol );
Stock_Info_Head->Type = Type;
Stock_Info_Head->Entry_Count = Entry_Count;
Stock_Info_Head->Price_High = Price_High;
Stock_Info_Head->Price_Low = Price_Low;
Stock_Info_Head->First_Ptr = Stock_Data_Head;
Stock_Info_Head->Last_Ptr = curr_ptr;
url_fclose(handle);
return (NULL); /* all done */
} /* End Type Price Data Block */
else
/******************************************************************************/
/* */
/* Read The Dividend Data: */
/* */
/******************************************************************************/
if ( Type == 'v' )
{ /* Start Type Dividend Data Block */
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Dividend Read\n" );
div_curr_ptr = Stock_Div_Head;
div_prev_ptr = NULL;
First_Record = TRUE;
Entry_Count = 0;
while( !url_feof( handle ))
{ /* Start While */
url_fgets( buffer, sizeof( buffer ), handle);
if ( Print_Head->Print_Data == TRUE )
printf ( "%s", buffer );
if ( First_Record == FALSE )
{ /* Start Not First_Record */
sscanf ( buffer, "%4d", &year );
sscanf ( buffer+5, "%2d", &month );
sscanf ( buffer+8, "%2d", &day );
sscanf ( buffer+11, "%f", ÷nd );
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Curr_Ptr %x\n", div_curr_ptr );
printf ( "Date: %4i %2i %2i\n", year, month, day );
printf ( "Dividend %2.2f\n", dividend );
};
div_curr_ptr->year = year;
div_curr_ptr->month = month;
div_curr_ptr->day = day;
div_curr_ptr->Dividend = dividend;
div_new_ptr = malloc( sizeof( Stock_Div_Struct) );
div_curr_ptr->next_ptr = div_new_ptr;
div_curr_ptr->prev_ptr = div_prev_ptr;
div_prev_ptr = div_curr_ptr;
div_curr_ptr = div_new_ptr;
Entry_Count++;
}; /* End Not First_Record */
First_Record = FALSE;
}; /* End While */
if ( Entry_Count != 0 )
div_curr_ptr = div_prev_ptr;
div_curr_ptr->next_ptr = NULL;
Stock_Info_Head->Date_Start.Day = Date_Start.Day;
Stock_Info_Head->Date_Start.Month = Date_Start.Month;
Stock_Info_Head->Date_Start.Year= Date_Start.Year;
Stock_Info_Head->Date_End.Day = Date_End.Day;
Stock_Info_Head->Date_End.Month = Date_End.Month;
Stock_Info_Head->Date_End.Year = Date_End.Year;
strcpy ( Stock_Info_Head->Symbol, Symbol );
Stock_Info_Head->Type = Type;
Stock_Info_Head->Entry_Count = Entry_Count;
Stock_Info_Head->First_Ptr = Stock_Div_Head;
Stock_Info_Head->Last_Ptr = div_curr_ptr;
url_fclose(handle);
return (NULL); /* all done */
} /* End Type Dividend Data Block */
};
/******************************************************************************/
/* */
/* Function: Read_Yahoo_Secondary */
/* */
/******************************************************************************/
int Read_Yahoo_Secondary ( Stock_Data_Struct *Stock_Data_Head, Stock_Div_Struct
*Stock_Div_Head, Stock_Data_Info_Struct *Stock_Info_Head, Date_Struct
Date_Start, Date_Struct Date_End, char Symbol[10], char Type, Print_Struct
*Print_Head )
{
URL_FILE *handle;
FILE *outf;
FILE *Test_Handle;
int nread;
char buffer[256]; // Was 256
char Char_Temp[16];
const char *url;
char URL_String[256];
char Temp[256];
int year;
int month;
int day;
float open_read;
float high_read;
float low_read;
float close_read;
long long int volume_read;
float adj_close_read;
float open;
float high;
float low;
float close;
long long int volume;
float adj_close;
float dividend;
Stock_Data_Struct *curr_ptr;
Stock_Data_Struct *new_ptr;
Stock_Data_Struct *prev_ptr;
Stock_Div_Struct *div_curr_ptr;
Stock_Div_Struct *div_new_ptr;
Stock_Div_Struct *div_prev_ptr;
int First_Record;
int Entry_Count;
float Price_High;
float Price_Low;
float Split_Divisor;
float Divisor;
float Int_Divisor;
/******************************************************************************/
/* */
/* Print The Input Variables: */
/* */
/******************************************************************************/
if ( Print_Head->Print_Headers_Read == TRUE )
{
printf ( "Running Read_Yahoo\n") ;
printf ( "Start Date: Month = %d\n", Date_Start.Month );
printf ( "Start Date: Day = %d\n", Date_Start.Day );
printf ( "Start Date: Year = %d\n", Date_Start.Year );
printf ( "End Date: Month = %d\n", Date_End.Month );
printf ( "End Date: Day = %d\n", Date_End.Day );
printf ( "End Date: Year = %d\n", Date_End.Year );
printf ( "Symbol = %s\n", Symbol );
printf ( "Type = %c\n", Type );
}
/******************************************************************************/
/* */
/* Assemble The URL: */
/* */
/******************************************************************************/
strcpy ( URL_String, "http://ichart.finance.yahoo.com/table.csv?s="; );
strcat ( URL_String, Symbol );
strcat ( URL_String, "&a=" );
sprintf ( Temp, "%d", Date_Start.Month-1);
strcat ( URL_String, Temp );
strcat ( URL_String, "&b=" );
sprintf ( Temp, "%d", Date_Start.Day );
strcat ( URL_String, Temp );
strcat ( URL_String, "&c=" );
sprintf ( Temp, "%d", Date_Start.Year);
strcat ( URL_String, Temp );
strcat ( URL_String, "&g=" );
sprintf ( Temp, "%c", Type );
strcat ( URL_String, Temp );
strcat ( URL_String, "&d=" );
sprintf ( Temp, "%d", Date_End.Month-1);
strcat ( URL_String, Temp );
strcat ( URL_String, "&e=" );
sprintf ( Temp, "%d", Date_End.Day);
strcat ( URL_String, Temp );
strcat ( URL_String, "&f=" );
sprintf ( Temp, "%d", Date_End.Year);
strcat ( URL_String, Temp );
strcat ( URL_String, "&ignore=.csv" );
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "URL String = %s\n", URL_String );
/*
url="http://ichart.finance.yahoo.com/table.csv?s=SPY&d=10&e=2&f=2012&g=d&a=0&b=29&c=1993&ignore=.csv";;
*/
/*
url="http://ichart.finance.yahoo.com/table.csv?s=SPY&d=10&e=2&f=2012&g=d&a=0&b=29&c=1993&ignore=.csv";;
*/
/******************************************************************************/
/* */
/* Read The Data From The Yahoo Web Site: */
/* */
/******************************************************************************/
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Yahoo Web Site Read\n" );
handle = url_fopen ( URL_String, "r" );
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Handle = %d\n", handle );
if( !handle )
{
printf( "Couldn't Open URL: url_fopen() %s\n", url );
printf ( "Handle = %d\n", handle );
fclose( outf );
return -1;
}
/******************************************************************************/
/* */
/* Read The Price / Volume Data: */
/* */
/******************************************************************************/
if ( Type == 'd' || Type == 'w' || Type == 'm' )
{ /* Start Type Price Data Block */
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Price Volume Read\n" );
curr_ptr = Stock_Data_Head;
prev_ptr = Stock_Data_Head;
prev_ptr->high = 1000000.0;
First_Record = TRUE;
Entry_Count = 0;
Price_High = 0.0;
Price_Low = 1000000000.0;
Split_Divisor = 1.0;
Divisor = 0.0;
Int_Divisor = 0.0;
while( !url_feof( handle ))
{ /* Start While */
url_fgets( buffer, sizeof( buffer ), handle);
if ( strstr ( buffer, "404 Not Found" ) != NULL )
{
printf( "Error 404 Symbol Not Found\n" );
fclose( outf );
return ( -1 );
}
if ( Print_Head->Print_Data == TRUE )
printf ( "%s", buffer );
if ( First_Record == FALSE )
{ /* Start Not First_Record */
sscanf ( buffer, "%4d", &year );
sscanf ( buffer+5, "%2d", &month );
sscanf ( buffer+8, "%2d", &day );
sscanf ( buffer+11, "%f,%f,%f,%f,%lli,%f", &open_read, &high_read,
&low_read, &close_read, &volume_read, &adj_close_read );
if ( ( 1.40 * Split_Divisor * prev_ptr->high ) <= high_read )
{ /* Start Block Split Occured */
Divisor = ( high_read / (Split_Divisor*prev_ptr->high));
if ( Divisor >= 7.8 )
Int_Divisor = 8;
else
if ( Divisor >= 6.8 )
Int_Divisor = 7;
else
if ( Divisor >= 5.8 )
Int_Divisor = 6;
else
if ( Divisor >= 4.8 )
Int_Divisor = 5;
else
if ( Divisor >= 3.8 )
Int_Divisor = 4;
else
if ( Divisor >= 2.8 )
Int_Divisor = 3;
else
if ( Divisor >= 1.6 )
Int_Divisor = 2;
else
if ( Divisor >= 1.3 )
Int_Divisor = 1.5;
Split_Divisor = (float) Split_Divisor * Int_Divisor;
curr_ptr->Split_Occured = TRUE;
if ( Print_Head->Print_Data == TRUE )
printf ( "Split Detected: Date: %d/%d/%d Int_Divisor %d
Split_Divisor %f\n", month, day, year, Int_Divisor, Split_Divisor);
} /* End Block Split Occured */
else
{ /* Start Block No Split Occured */
Int_Divisor = 0.0;
curr_ptr->Split_Occured = FALSE;
}; /* End Block No Split Occured */
open = open_read / Split_Divisor;
high = high_read / Split_Divisor;
low = low_read / Split_Divisor;
close = close_read / Split_Divisor;
volume = volume_read;
adj_close = adj_close_read / Split_Divisor;
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Curr_Ptr %x\n", curr_ptr );
printf ( "Date: %4i %2i %2i\n", year, month, day );
printf ( "Open: %2.2f\n", open);
printf ( "High: %2.2f\n", high );
printf ( "Low: %2.2f\n", low );
printf ( "Close: %2.2f\n", close );
printf ( "Volume: %i\n", volume );
printf ( "Adj_Close: %2.2f\n", adj_close );
printf ( "High_Read: %2.2f\n", high_read);
printf ( "Prev High: %2.2f\n", prev_ptr->high );
printf ( "Divisor: %2.2f\n", Divisor );
printf ( "Split_Divisor: %2.2f\n", Split_Divisor );
printf ( "Int_Divisor: %2.2f\n\n", Int_Divisor );
};
curr_ptr->year = year;
curr_ptr->month = month;
curr_ptr->day = day;
curr_ptr->open = open;
curr_ptr->high = high;
curr_ptr->low = low;
curr_ptr->close = close;
curr_ptr->volume = volume;
curr_ptr->adj_close = adj_close;
curr_ptr->open_read = open_read;
curr_ptr->high_read = high_read;
curr_ptr->low_read = low_read;
curr_ptr->close_read = close_read;
curr_ptr->volume_read = volume_read;
curr_ptr->adj_close_read = adj_close_read;
curr_ptr->Int_Divisor = Int_Divisor;
curr_ptr->Split_Divisor = Split_Divisor;
curr_ptr->Entry_Count = Entry_Count;
if ( high > Price_High )
Price_High = high;
if ( low < Price_Low )
Price_Low = low;
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Price_High = %f\n", Price_High );
printf ( "Price_Low = %f\n", Price_Low );
};
new_ptr = malloc( sizeof( Stock_Data_Struct) );
curr_ptr->next_ptr = new_ptr;
curr_ptr->prev_ptr = prev_ptr;
prev_ptr = curr_ptr;
curr_ptr = new_ptr;
Entry_Count++;
}; /* End Not First_Record */
First_Record = FALSE;
}; /* End While */
if ( Entry_Count != NULL )
curr_ptr = prev_ptr;
curr_ptr->next_ptr = NULL;
Stock_Info_Head->Date_Start.Day = Date_Start.Day;
Stock_Info_Head->Date_Start.Month = Date_Start.Month;
Stock_Info_Head->Date_Start.Year= Date_Start.Year;
Stock_Info_Head->Date_End.Day = Date_End.Day;
Stock_Info_Head->Date_End.Month = Date_End.Month;
Stock_Info_Head->Date_End.Year = Date_End.Year;
strcpy ( Stock_Info_Head->Symbol, Symbol );
Stock_Info_Head->Type = Type;
Stock_Info_Head->Entry_Count = Entry_Count;
Stock_Info_Head->Price_High = Price_High;
Stock_Info_Head->Price_Low = Price_Low;
Stock_Info_Head->First_Ptr = Stock_Data_Head;
Stock_Info_Head->Last_Ptr = curr_ptr;
url_fclose(handle);
return (NULL); /* all done */
} /* End Type Price Data Block */
else
/******************************************************************************/
/* */
/* Read The Dividend Data: */
/* */
/******************************************************************************/
if ( Type == 'v' )
{ /* Start Type Dividend Data Block */
if ( Print_Head->Print_Headers_Read == TRUE )
printf ( "Start The Dividend Read\n" );
div_curr_ptr = Stock_Div_Head;
div_prev_ptr = NULL;
First_Record = TRUE;
Entry_Count = 0;
while( !url_feof( handle ))
{ /* Start While */
url_fgets( buffer, sizeof( buffer ), handle);
if ( Print_Head->Print_Data == TRUE )
printf ( "%s", buffer );
if ( First_Record == FALSE )
{ /* Start Not First_Record */
sscanf ( buffer, "%4d", &year );
sscanf ( buffer+5, "%2d", &month );
sscanf ( buffer+8, "%2d", &day );
sscanf ( buffer+11, "%f", ÷nd );
if ( Print_Head->Print_Data == TRUE )
{
printf ( "Curr_Ptr %x\n", div_curr_ptr );
printf ( "Date: %4i %2i %2i\n", year, month, day );
printf ( "Dividend %2.2f\n", dividend );
};
div_curr_ptr->year = year;
div_curr_ptr->month = month;
div_curr_ptr->day = day;
div_curr_ptr->Dividend = dividend;
div_new_ptr = malloc( sizeof( Stock_Div_Struct) );
div_curr_ptr->next_ptr = div_new_ptr;
div_curr_ptr->prev_ptr = div_prev_ptr;
div_prev_ptr = div_curr_ptr;
div_curr_ptr = div_new_ptr;
Entry_Count++;
}; /* End Not First_Record */
First_Record = FALSE;
}; /* End While */
if ( Entry_Count != 0 )
div_curr_ptr = div_prev_ptr;
div_curr_ptr->next_ptr = NULL;
Stock_Info_Head->Date_Start.Day = Date_Start.Day;
Stock_Info_Head->Date_Start.Month = Date_Start.Month;
Stock_Info_Head->Date_Start.Year= Date_Start.Year;
Stock_Info_Head->Date_End.Day = Date_End.Day;
Stock_Info_Head->Date_End.Month = Date_End.Month;
Stock_Info_Head->Date_End.Year = Date_End.Year;
strcpy ( Stock_Info_Head->Symbol, Symbol );
Stock_Info_Head->Type = Type;
Stock_Info_Head->Entry_Count = Entry_Count;
Stock_Info_Head->First_Ptr = Stock_Div_Head;
Stock_Info_Head->Last_Ptr = div_curr_ptr;
url_fclose(handle);
return (NULL); /* all done */
} /* End Type Dividend Data Block */
};
/******************************************************************************/
/* */
/* Function: write_callback */
/* */
/******************************************************************************/
/* curl calls this routine to get more data */
static size_t write_callback(char *buffer,
size_t size,
size_t nitems,
void *userp)
{
char *newbuff;
size_t rembuff;
URL_FILE *url = (URL_FILE *)userp;
size *= nitems;
rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */
if(size > rembuff) {
/* not enough space in buffer */
newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
if(newbuff==NULL) {
fprintf(stderr,"callback buffer grow failed\n");
size=rembuff;
}
else {
/* realloc suceeded increase buffer size*/
url->buffer_len+=size - rembuff;
url->buffer=newbuff;
}
}
memcpy(&url->buffer[url->buffer_pos], buffer, size);
url->buffer_pos += size;
return size;
}
/* use to attempt to fill the read buffer up to requested number of bytes */
/******************************************************************************/
/* */
/* Function: fill_buffer */
/* */
/******************************************************************************/
static int fill_buffer(URL_FILE *file, size_t want)
{
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
struct timeval timeout;
int rc;
/* only attempt to fill buffer if transactions still running and buffer
* doesnt exceed required size already
*/
if((!file->still_running) || (file->buffer_pos > want))
return 0;
/* attempt to fill buffer */
do {
int maxfd = -1;
long curl_timeo = -1;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
/* set a suitable timeout to fail on */
timeout.tv_sec = 60; /* 1 minute */
timeout.tv_usec = 0;
curl_multi_timeout(multi_handle, &curl_timeo);
if(curl_timeo >= 0) {
timeout.tv_sec = curl_timeo / 1000;
if(timeout.tv_sec > 1)
timeout.tv_sec = 1;
else
timeout.tv_usec = (curl_timeo % 1000) * 1000;
}
/* get file descriptors from the transfers */
curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
/* In a real-world program you OF COURSE check the return code of the
function calls. On success, the value of maxfd is guaranteed to be
greater or equal than -1. We call select(maxfd + 1, ...), specially
in case of (maxfd == -1), we call select(0, ...), which is basically
equal to sleep. */
rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
switch(rc) {
case -1:
/* select error */
break;
case 0:
default:
/* timeout or readable/writable sockets */
curl_multi_perform(multi_handle, &file->still_running);
break;
}
} while(file->still_running && (file->buffer_pos < want));
return 1;
}
/* use to remove want bytes from the front of a files buffer */
/******************************************************************************/
/* */
/* Function: use_buffer */
/* */
/******************************************************************************/
static int use_buffer(URL_FILE *file,int want)
{
/* sort out buffer */
if((file->buffer_pos - want) <=0) {
/* ditch buffer - write will recreate */
if(file->buffer)
free(file->buffer);
file->buffer=NULL;
file->buffer_pos=0;
file->buffer_len=0;
}
else {
/* move rest down make it available for later */
memmove(file->buffer,
&file->buffer[want],
(file->buffer_pos - want));
file->buffer_pos -= want;
}
return 0;
}
URL_FILE *url_fopen(const char *url,const char *operation)
{
/* this code could check for URLs or types in the 'url' and
basicly use the real fopen() for standard files */
URL_FILE *file;
(void)operation;
file = malloc(sizeof(URL_FILE));
if(!file)
return NULL;
memset(file, 0, sizeof(URL_FILE));
if((file->handle.file=fopen(url,operation)))
file->type = CFTYPE_FILE; /* marked as URL */
else
{
file->type = CFTYPE_CURL; /* marked as URL */
file->handle.curl = curl_easy_init();
curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
if(!multi_handle)
multi_handle = curl_multi_init();
curl_multi_add_handle(multi_handle, file->handle.curl);
/* lets start the fetch */
curl_multi_perform(multi_handle, &file->still_running);
if((file->buffer_pos == 0) && (!file->still_running)) {
/* if still_running is 0 now, we should return NULL */
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(multi_handle, file->handle.curl);
/* cleanup */
curl_easy_cleanup(file->handle.curl);
free(file);
file = NULL;
}
}
return file;
}
/******************************************************************************/
/* */
/* Function: url_fclose */
/* */
/******************************************************************************/
int url_fclose(URL_FILE *file)
{
int ret=0;/* default is good return */
switch(file->type) {
case CFTYPE_FILE:
ret=fclose(file->handle.file); /* passthrough */
break;
case CFTYPE_CURL:
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(multi_handle, file->handle.curl);
/* cleanup */
curl_easy_cleanup(file->handle.curl);
break;
default: /* unknown or supported type - oh dear */
ret=EOF;
errno=EBADF;
break;
}
if(file->buffer)
free(file->buffer);/* free any allocated buffer space */
free(file);
return ret;
}
/******************************************************************************/
/* */
/* Function: url_feof */
/* */
/******************************************************************************/
int url_feof(URL_FILE *file)
{
int ret=0;
switch(file->type) {
case CFTYPE_FILE:
ret=feof(file->handle.file);
break;
case CFTYPE_CURL:
if((file->buffer_pos == 0) && (!file->still_running))
ret = 1;
break;
default: /* unknown or supported type - oh dear */
ret=-1;
errno=EBADF;
break;
}
return ret;
}
/******************************************************************************/
/* */
/* Function: url_fread */
/* */
/******************************************************************************/
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
{
size_t want;
switch(file->type) {
case CFTYPE_FILE:
want=fread(ptr,size,nmemb,file->handle.file);
break;
case CFTYPE_CURL:
want = nmemb * size;
fill_buffer(file,want);
/* check if theres data in the buffer - if not fill_buffer()
* either errored or EOF */
if(!file->buffer_pos)
return 0;
/* ensure only available data is considered */
if(file->buffer_pos < want)
want = file->buffer_pos;
/* xfer data to caller */
memcpy(ptr, file->buffer, want);
use_buffer(file,want);
want = want / size; /* number of items */
break;
default: /* unknown or supported type - oh dear */
want=0;
errno=EBADF;
break;
}
return want;
}
/******************************************************************************/
/* */
/* Function: url_fgets */
/* */
/******************************************************************************/
char *url_fgets(char *ptr, size_t size, URL_FILE *file)
{
size_t want = size - 1;/* always need to leave room for zero termination */
size_t loop;
switch(file->type) {
case CFTYPE_FILE:
ptr = fgets(ptr,size,file->handle.file);
break;
case CFTYPE_CURL:
fill_buffer(file,want);
/* check if theres data in the buffer - if not fill either errored or
* EOF */
if(!file->buffer_pos)
return NULL;
/* ensure only available data is considered */
if(file->buffer_pos < want)
want = file->buffer_pos;
/*buffer contains data */
/* look for newline or eof */
for(loop=0;loop < want;loop++) {
if(file->buffer[loop] == '\n') {
want=loop+1;/* include newline */
break;
}
}
/* xfer data to caller */
memcpy(ptr, file->buffer, want);
ptr[want]=0;/* allways null terminate */
use_buffer(file,want);
break;
default: /* unknown or supported type - oh dear */
ptr=NULL;
errno=EBADF;
break;
}
return ptr;/*success */
}
/******************************************************************************/
/* */
/* Function: url_rewind */
/* */
/******************************************************************************/
void url_rewind(URL_FILE *file)
{
switch(file->type) {
case CFTYPE_FILE:
rewind(file->handle.file); /* passthrough */
break;
case CFTYPE_CURL:
/* halt transaction */
curl_multi_remove_handle(multi_handle, file->handle.curl);
/* restart */
curl_multi_add_handle(multi_handle, file->handle.curl);
/* ditch buffer - write will recreate - resets stream pos*/
if(file->buffer)
free(file->buffer);
file->buffer=NULL;
file->buffer_pos=0;
file->buffer_len=0;
break;
default: /* unknown or supported type - oh dear */
break;
}
}
-------------------------------------------------------------------
List admin: http://cool.haxx.se/list/listinfo/curl-library
Etiquette: http://curl.haxx.se/mail/etiquette.html
