Hi Drake, Hi Richard, Hi Paul,
Thanks for all your comments, I really appreciate the help.
Please see FULL code below. You can literally copy and paste this and
compile it with something like:
gcc -m64 -Wall -g -I./ -c ./main.c -o ./bigbyte.o
gcc -o ./bigbyte ./bigbyte.o -L./ -Wl,-rpath,./ -lsqlite3 -lpthread -ldl
Then if you test it with a file called "in.txt" that has a NUL in it you
will see that it cuts it off in the SQLite db "test_queue"....
Can you perhaps see something I am doing wrong below? C is not my day
job ......as you can see ;-)
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "sqlite3.h"
#include <time.h>
typedef struct messageStruct {
char tid[37];
void *raw_stream_in;
int num_bytes_in;
void *raw_stream_out;
int num_bytes_out;
char timestamp[20];
}message;
sqlite3_stmt *stmt;
static sqlite3 *handle;
int rc;
void* p;
char *queue;
int queue_createQueue(char *queueName);
int queue_openQueue(char *queueName);
int queue_closeQueue();
int queue_send(message msg);
int main(){
char *buffer = NULL;
time_t t;
struct tm *tmp;
FILE *fp;
if((fp = fopen("./in.txt", "rb"))==NULL){
printf("Cannot open file2\n");
exit(1);
}
fseek(fp, 0, SEEK_END);
long int fsize = ftell(fp);
printf("File size: %i\n", fsize);
rewind(fp);
buffer = malloc(fsize * sizeof(char));
fread(buffer, 1, fsize, fp);
fclose(fp);
//Create Queue
rc = queue_createQueue("test_queue");
if (rc ==0){
printf("Queue created...\n");
}
message msg_in = {};
strcpy(msg_in.tid, "123");
msg_in.tid[36] = '\0';
msg_in.raw_stream_in = buffer;
msg_in.num_bytes_in = fsize;
msg_in.raw_stream_out = NULL;
msg_in.num_bytes_out = 0;
t = time(NULL);
tmp = localtime(&t);
if (tmp == NULL) {
exit(1);
}
strftime(msg_in.timestamp, sizeof(msg_in.timestamp), "%Y-%m-%d
%H:%M:%S", tmp);
rc = queue_send(msg_in);
if(rc == 0){
printf("Message sent to queue successfully\n");
}
rc = queue_closeQueue();
if (rc ==0){
printf("Queue closed...\n");
}
return 0;
}
//Queue handler while waiting to get queue lock
int queueBusyHandler(void* db, int count){
return 1;
}
//Create Queue
int queue_createQueue(char *queueName){
char *queueTable = "CREATE TABLE [queue] ( "
"[idx] INTEGER NOT NULL PRIMARY KEY
AUTOINCREMENT, "
"[tid] CHAR(24) NOT NULL, "
"[raw_stream_in] BLOB, "
"[num_bytes_in] INTEGER "
"[raw_stream_out] BLOB, "
"[num_bytes_out] INTEGER "
"[timestamp] CHAR(20));";
char *index = "CREATE UNIQUE INDEX tid on queue([tid]);";
char *journalMode = "PRAGMA journal_mode=wal;";
int successFlag = 1;
rc = sqlite3_open_v2(queueName,&handle, SQLITE_OPEN_READWRITE |
SQLITE_OPEN_SHAREDCACHE | SQLITE_OPEN_CREATE, NULL);
if (rc == 0){
p = sqlite3_malloc(256);
sqlite3_busy_handler(handle, &queueBusyHandler, p);
sqlite3_free(p);
sqlite3_exec(handle,"PRAGMA default_cache_size = 50;",0,0,0);
rc = sqlite3_exec(handle,journalMode,0,0,0);
if(rc == 0){
rc = sqlite3_exec(handle,queueTable,0,0,0);
if(rc == 0){
rc = sqlite3_exec(handle,index,0,0,0);
if (rc == 0){
successFlag = 0;
}
}
}
}
return successFlag;
}
//Close Queue
int queue_closeQueue(){
rc = sqlite3_close(handle);
return rc;
}
//Send
int queue_send(message msg){
int idx;
char *sendText = "INSERT INTO queue (tid, raw_stream_in,
num_bytes_in) "
"VALUES(:tid, :raw_stream_in, :num_bytes_in)";
sqlite3_exec(handle,"BEGIN IMMEDIATE TRANSACTION;",0,0,0);
sqlite3_prepare_v2(handle,sendText,-1,&stmt,0 );
idx = sqlite3_bind_parameter_index( stmt, ":tid" );
sqlite3_bind_text( stmt, idx, msg.tid, -1, SQLITE_STATIC );
idx = sqlite3_bind_parameter_index( stmt, ":raw_stream_in" );
sqlite3_bind_blob( stmt, idx, msg.raw_stream_in, msg.num_bytes_in,
SQLITE_TRANSIENT );
idx = sqlite3_bind_parameter_index( stmt, ":num_bytes_in" );
sqlite3_bind_int( stmt, idx, msg.num_bytes_in);
rc = sqlite3_step(stmt);
rc = sqlite3_exec(handle,"COMMIT TRANSACTION;",0,0,0);
rc = sqlite3_finalize(stmt);
return 0;
}
On 03/04/2011 13:11, Drake Wilson wrote:
> Quoth Lynton Grice<[email protected]>, on 2011-04-03 12:37:06 +0200:
>> There are NULL characters
> You mean NUL characters.
>
>> in the first couple header fields [...]
>>
>> I am no C expert but I have the following table defined:
>>
>> char *queueTable = "CREATE TABLE [test] ( "
>> "[idx] INTEGER NOT NULL PRIMARY KEY
>> AUTOINCREMENT, "
>> "[raw_stream_in] BLOB, "
>> "[num_bytes_in] INTEGER );";
> I wouldn't recommend the use of [] for quoting identifiers (I'd prefer
> double-quotes for more ANSI-esque SQL), but that looks like it should
> work fine.
>
>> sqlite3_bind_blob( stmt, idx, msg.raw_stream_in, msg.num_bytes_in,
>> SQLITE_STATIC );
> Be careful with SQLITE_STATIC. AIUI, the lifetime of the buffer must
> then be a superset of the lifetime of the statement.
>
>> For testing purposes I simply read the 15MB file from file into a char *.
>>
>> char *buffer = NULL;
>> FILE *fp;
>> if((fp = fopen("./in.txt", "r"))==NULL){
>> exit(1);
>> }
> Not "rb" for binary mode?
>
>> fseek(fp, 0, SEEK_END);
>> long int fsize = ftell(fp);
>> printf("File size: %i\n", fsize);
>> rewind(fp);
>> buffer = malloc(fsize * sizeof(char));
>> fread(buffer, 1, fsize, fp);
>> buffer[fsize] = '\0';
> You're overwriting beyond the end of the array. You don't need an
> extra NUL here.
>
>> fclose(fp);
>>
>> I then point the MSG "raw_stream_in" to the buffer:
>>
>> msg_in.raw_stream_in = buffer;
> And you set msg.num_bytes_in where?
>
>> I then use the following statement as mentioned before to insert the
>> stream into the BLOB field:
>>
>> sqlite3_bind_blob( stmt, idx, msg.raw_stream_in, msg.num_bytes_in,
>> SQLITE_STATIC );
> That looks fine by itself, subject to the caveats of SQLITE_STATIC
> above and whether stmt and idx are valid.
>
>> _So essentially all characters are copied until the first NULL
>> char._
> No. I strongly suspect that's a red herring.
>
> In summary:
>
> - Make sure msg.num_bytes_in is actually set to what you want.
> - Make sure you're handling the lifetime of the buffer correctly;
> for testing purposes I'd use SQLITE_TRANSIENT rather than
> SQLITE_STATIC, since that evades that issue at the cost of a
> memcpy.
> - Don't write past the end of the array.
>
> ---> Drake Wilson
> _______________________________________________
> sqlite-users mailing list
> [email protected]
> http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-users
_______________________________________________
sqlite-users mailing list
[email protected]
http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-users
