RPM Package Manager, CVS Repository http://rpm5.org/cvs/ ____________________________________________________________________________
Server: rpm5.org Name: Jeff Johnson Root: /v/rpm/cvs Email: [email protected] Module: rpm Date: 04-Apr-2011 00:10:52 Branch: HEAD Handle: 2011040322105100 Modified files: rpm CHANGES rpm/rpmio librpmio.vers mongo.c Log: - mongo: and re-export the mongo-c-driver through librpmio.vers. - mongo: pull in the gridfs methods from the mongo-c-driver too. Summary: Revision Changes Path 1.3599 +2 -0 rpm/CHANGES 2.202 +34 -0 rpm/rpmio/librpmio.vers 2.5 +1058 -0 rpm/rpmio/mongo.c ____________________________________________________________________________ patch -p0 <<'@@ .' Index: rpm/CHANGES ============================================================================ $ cvs diff -u -r1.3598 -r1.3599 CHANGES --- rpm/CHANGES 3 Apr 2011 21:18:04 -0000 1.3598 +++ rpm/CHANGES 3 Apr 2011 22:10:51 -0000 1.3599 @@ -1,4 +1,6 @@ 5.4.0 -> 5.4.1: + - jbj: mongo: and re-export the mongo-c-driver through librpmio.vers. + - jbj: mongo: pull in the gridfs methods from the mongo-c-driver too. - jbj: mongo: update mongo-c-driver code, gridfs stil todo++. - jbj: mongo: stub-in a configurable mongodb:// %_mongodb URI. - jbj: autofu: fix: remove --with-js remnants. @@ . patch -p0 <<'@@ .' Index: rpm/rpmio/librpmio.vers ============================================================================ $ cvs diff -u -r2.201 -r2.202 librpmio.vers --- rpm/rpmio/librpmio.vers 29 Dec 2010 16:33:53 -0000 2.201 +++ rpm/rpmio/librpmio.vers 3 Apr 2011 22:10:51 -0000 2.202 @@ -791,6 +791,7 @@ bson_append_string_base; bson_append_symbol; bson_append_time_t; + bson_append_timestamp; bson_append_undefined; bson_buffer_destroy; bson_buffer_finish; @@ -830,6 +831,7 @@ bson_iterator_subiterator; bson_iterator_subobject; bson_iterator_time_t; + bson_iterator_timestamp; bson_iterator_type; bson_iterator_value; bson_malloc; @@ -840,10 +842,39 @@ bson_oid_to_string; bson_print; bson_print_raw; + bson_realloc; bson_size; bson_swap_endian64; bson_swap_endian32; set_bson_err_handler; + gridfile_destroy; + gridfile_exists; + gridfile_get_boolean; + gridfile_get_chunk; + gridfile_get_chunks; + gridfile_get_chunksize; + gridfile_get_contentlength; + gridfile_get_contenttype; + gridfile_get_field; + gridfile_get_filename; + gridfile_get_md5; + gridfile_get_metadata; + gridfile_get_numchunks; + gridfile_get_uploaddate; + gridfile_init; + gridfile_read; + gridfile_seek; + gridfile_write_buffer; + gridfile_write_file; + gridfile_writer_done; + gridfile_writer_init; + gridfs_destroy; + gridfs_find_filename; + gridfs_find_query; + gridfs_init; + gridfs_remove_filename; + gridfs_store_buffer; + gridfs_store_file; mongo_cmd_add_user; mongo_cmd_authenticate; mongo_cmd_drop_collection; @@ -877,6 +908,9 @@ mongo_read_response; mongo_reconnect; mongo_remove; + mongo_replset_init_conn; + mongo_replset_add_seed; + mongo_replset_connect; mongo_run_command; mongo_simple_int_command; mongo_simple_str_command; @@ . patch -p0 <<'@@ .' Index: rpm/rpmio/mongo.c ============================================================================ $ cvs diff -u -r2.4 -r2.5 mongo.c --- rpm/rpmio/mongo.c 3 Apr 2011 21:18:04 -0000 2.4 +++ rpm/rpmio/mongo.c 3 Apr 2011 22:10:51 -0000 2.5 @@ -40,6 +40,1064 @@ static const int one = 1; /*==============================================================*/ +/* --- gridfs.h */ + +enum {DEFAULT_CHUNK_SIZE = 256 * 1024}; + +typedef uint64_t gridfs_offset; + +/* A GridFS contains a db connection, a root database name, and an + optional prefix */ +typedef struct { + /* The client to db-connection. */ + mongo_connection* client; + /* The root database name */ + const char* dbname; + /* The prefix of the GridFS's collections, default is NULL */ + const char* prefix; + /* The namespace where the file's metadata is stored */ + const char* files_ns; + /* The namespace where the files's data is stored in chunks */ + const char* chunks_ns; + +} gridfs; + +/* The state of a gridfile. This is used for incrementally writing buffers + * to a single GridFS file. + */ + +/* A GridFile contains the GridFS it is located in and the file + metadata */ +typedef struct { + /* The GridFS where the GridFile is located */ + gridfs* gfs; + /* The GridFile's bson object where all its metadata is located */ + bson* meta; + /* The position is the offset in the file */ + gridfs_offset pos; + /* The files_id of the gridfile */ + bson_oid_t id; + /* The name of the gridfile as a string */ + const char* remote_name; + /* The gridfile's content type */ + const char* content_type; + /* The length of this gridfile */ + gridfs_offset length; + /* The number of the current chunk being written to */ + int chunk_num; + /* A buffer storing data still to be written to chunks */ + char* pending_data; + /* Length of pending data */ + int pending_len; + +} gridfile; + +/*--------------------------------------------------------------------*/ + +/** Initializes a GridFS object + * @param client - db connection + * @param dbname - database name + * @param prefix - collection prefix, default is fs if NULL or empty + * @param gfs - the GridFS object to initialize + * @return - 1 if successful, 0 otherwise + */ +int gridfs_init(mongo_connection* client, const char* dbname, + const char* prefix, gridfs* gfs); + +/** Destroys a GridFS object + */ +void gridfs_destroy( gridfs* gfs ); + +/** Initializes a gridfile for writing incrementally with gridfs_write_buffer. + * Once initialized, you can write any number of buffers with gridfs_write_buffer. + * When done, you must call gridfs_writer_done to save the file metadata. + * + * @return - 1 if successful, 0 otherwise + */ +void gridfile_writer_init( gridfile* gfile, gridfs* gfs, const char* remote_name, const char* content_type ); + +/** Write to a GridFS file incrementally. You can call this function any number + * of times with a new buffer each time. This allows you to effectively + * stream to a GridFS file. When finished, be sure to call gridfs_writer_done. + * + * @return - 1 if successful, 0 otherwise + */ +void gridfile_write_buffer( gridfile* gfile, const char* data, gridfs_offset length ); + +/** Signal that writing of this gridfile is complete by + * writing any buffered chunks along with the entry in the + * files collection. + * + * @return - the file object if successful; otherwise 0. + */ +bson gridfile_writer_done( gridfile* gfile ); + +/** Store a buffer as a GridFS file. + * @param gfs - the working GridFS + * @param data - pointer to buffer to store in GridFS + * @param length - length of the buffer + * @param remotename - filename for use in the database + * @param contenttype - optional MIME type for this object + * @return - the file object + */ +bson gridfs_store_buffer(gridfs* gfs, const char* data, gridfs_offset length, + const char* remotename, + const char * contenttype); + +/** Open the file referenced by filename and store it as a GridFS file. + * @param gfs - the working GridFS + * @param filename - local filename relative to the process + * @param remotename - optional filename for use in the database + * @param contenttype - optional MIME type for this object + * @return - the file object + */ +bson gridfs_store_file(gridfs* gfs, const char* filename, + const char* remotename, const char* contenttype); + +/** Removes the files referenced by filename from the db + * @param gfs - the working GridFS + * @param filename - the filename of the file/s to be removed + */ +void gridfs_remove_filename(gridfs* gfs, const char* filename); + +/** Find the first query within the GridFS and return it as a GridFile + * @param gfs - the working GridFS + * @param query - a pointer to the bson with the query data + * @param gfile - the output GridFile to be initialized + * @return 1 if successful, 0 otherwise + */ +int gridfs_find_query(gridfs* gfs, bson* query, gridfile* gfile ); + +/** Find the first file referenced by filename within the GridFS + * and return it as a GridFile + * @param gfs - the working GridFS + * @param filename - filename of the file to find + * @param gfile - the output GridFile to be intialized + * @return 1 if successful, 0 otherwise + */ +int gridfs_find_filename(gridfs* gfs, const char *filename, + gridfile* gfile); + +/*--------------------------------------------------------------------*/ + + +/** Initializes a GridFile containing the GridFS and file bson + * @param gfs - the GridFS where the GridFile is located + * @param meta - the file object + * @param gfile - the output GridFile that is being initialized + * @return 1 if successful, 0 otherwise + */ +int gridfile_init(gridfs* gfs, bson* meta, gridfile* gfile); + +/** Destroys the GridFile + * @param oGridFIle - the GridFile being destroyed + */ +void gridfile_destroy(gridfile* gfile); + +/** Returns whether or not the GridFile exists + * @param gfile - the GridFile being examined + */ +int gridfile_exists(gridfile* gfile); + +/** Returns the filename of GridFile + * @param gfile - the working GridFile + * @return - the filename of the Gridfile + */ +const char * gridfile_get_filename(gridfile* gfile); + +/** Returns the size of the chunks of the GridFile + * @param gfile - the working GridFile + * @return - the size of the chunks of the Gridfile + */ +int gridfile_get_chunksize(gridfile* gfile); + +/** Returns the length of GridFile's data + * @param gfile - the working GridFile + * @return - the length of the Gridfile's data + */ +gridfs_offset gridfile_get_contentlength(gridfile* gfile); + +/** Returns the MIME type of the GridFile + * @param gfile - the working GridFile + * @return - the MIME type of the Gridfile + * (NULL if no type specified) + */ +const char* gridfile_get_contenttype(gridfile* gfile); + +/** Returns the upload date of GridFile + * @param gfile - the working GridFile + * @return - the upload date of the Gridfile + */ +bson_date_t gridfile_get_uploaddate(gridfile* gfile); + +/** Returns the MD5 of GridFile + * @param gfile - the working GridFile + * @return - the MD5 of the Gridfile + */ +const char* gridfile_get_md5(gridfile* gfile); + +/** Returns the field in GridFile specified by name + * @param gfile - the working GridFile + * @param name - the name of the field to be returned + * @return - the data of the field specified + * (NULL if none exists) + */ +const char *gridfile_get_field(gridfile* gfile, + const char* name); + +/** Returns a boolean field in GridFile specified by name + * @param gfile - the working GridFile + * @param name - the name of the field to be returned + * @return - the boolean of the field specified + * (NULL if none exists) + */ +bson_bool_t gridfile_get_boolean(gridfile* gfile, + const char* name); + +/** Returns the metadata of GridFile + * @param gfile - the working GridFile + * @return - the metadata of the Gridfile in a bson object + * (an empty bson is returned if none exists) + */ +bson gridfile_get_metadata(gridfile* gfile); + +/** Returns the number of chunks in the GridFile + * @param gfile - the working GridFile + * @return - the number of chunks in the Gridfile + */ +int gridfile_get_numchunks(gridfile* gfile); + +/** Returns chunk n of GridFile + * @param gfile - the working GridFile + * @return - the nth chunk of the Gridfile + */ +bson gridfile_get_chunk(gridfile* gfile, int n); + +/** Returns a mongo_cursor of *size* chunks starting with chunk *start* + * @param gfile - the working GridFile + * @param start - the first chunk in the cursor + * @param size - the number of chunks to be returned + * @return - mongo_cursor of the chunks (must be destroyed after use) + */ +mongo_cursor* gridfile_get_chunks(gridfile* gfile, int start, int size); + +/** Writes the GridFile to a stream + * @param gfile - the working GridFile + * @param stream - the file stream to write to + */ +gridfs_offset gridfile_write_file(gridfile* gfile, FILE* stream); + +/** Reads length bytes from the GridFile to a buffer + * and updates the position in the file. + * (assumes the buffer is large enough) + * (if size is greater than EOF gridfile_read reads until EOF) + * @param gfile - the working GridFile + * @param size - the amount of bytes to be read + * @param buf - the buffer to read to + * @return - the number of bytes read + */ +gridfs_offset gridfile_read(gridfile* gfile, gridfs_offset size, char* buf); + +/** Updates the position in the file + * (If the offset goes beyond the contentlength, + * the position is updated to the end of the file.) + * @param gfile - the working GridFile + * @param offset - the position to update to + * @return - resulting offset location + */ +gridfs_offset gridfile_seek(gridfile* gfile, gridfs_offset offset); + +/*==============================================================*/ +/* --- gridfs.c */ +#define TRUE 1 +#define FALSE 0 + +/*--------------------------------------------------------------------*/ + +static bson * chunk_new(bson_oid_t id, int chunkNumber, + const char * data, int len) + +{ + bson * b; + bson_buffer buf; + + b = (bson *)bson_malloc(sizeof(bson)); + if (b == NULL) return NULL; + + bson_buffer_init(&buf); + bson_append_oid(&buf, "files_id", &id); + bson_append_int(&buf, "n", chunkNumber); + bson_append_binary(&buf, "data", 2, data, len); + bson_from_buffer(b, &buf); + return b; +} + +/*--------------------------------------------------------------------*/ + +static void chunk_free(bson * oChunk) + +{ + bson_destroy(oChunk); + free(oChunk); +} + +/*--------------------------------------------------------------------*/ + +int gridfs_init(mongo_connection * client, const char * dbname, + const char * prefix, gridfs* gfs) +{ + int options; + bson_buffer bb; + bson b; + bson out; + bson_bool_t success; + + gfs->client = client; + + /* Allocate space to own the dbname */ + gfs->dbname = (const char *)bson_malloc(strlen(dbname)+1); + if (gfs->dbname == NULL) { + return FALSE; + } + strcpy((char*)gfs->dbname, dbname); + + /* Allocate space to own the prefix */ + if (prefix == NULL) prefix = "fs"; + gfs->prefix = (const char *)bson_malloc(strlen(prefix)+1); + if (gfs->prefix == NULL) { + free((char*)gfs->dbname); + return FALSE; + } + strcpy((char *)gfs->prefix, prefix); + + /* Allocate space to own files_ns */ + gfs->files_ns = + (const char *) bson_malloc (strlen(prefix)+strlen(dbname)+strlen(".files")+2); + if (gfs->files_ns == NULL) { + free((char*)gfs->dbname); + free((char*)gfs->prefix); + return FALSE; + } + strcpy((char*)gfs->files_ns, dbname); + strcat((char*)gfs->files_ns, "."); + strcat((char*)gfs->files_ns, prefix); + strcat((char*)gfs->files_ns, ".files"); + + /* Allocate space to own chunks_ns */ + gfs->chunks_ns = (const char *) bson_malloc(strlen(prefix) + strlen(dbname) + + strlen(".chunks") + 2); + if (gfs->chunks_ns == NULL) { + free((char*)gfs->dbname); + free((char*)gfs->prefix); + free((char*)gfs->files_ns); + return FALSE; + } + strcpy((char*)gfs->chunks_ns, dbname); + strcat((char*)gfs->chunks_ns, "."); + strcat((char*)gfs->chunks_ns, prefix); + strcat((char*)gfs->chunks_ns, ".chunks"); + + bson_buffer_init(&bb); + bson_append_int(&bb, "filename", 1); + bson_from_buffer(&b, &bb); + options = 0; + success = mongo_create_index(gfs->client, gfs->files_ns, &b, options, &out); + bson_destroy(&b); + if (!success) { + free((char*)gfs->dbname); + free((char*)gfs->prefix); + free((char*)gfs->files_ns); + free((char*)gfs->chunks_ns); + return FALSE; + } + + bson_buffer_init(&bb); + bson_append_int(&bb, "files_id", 1); + bson_append_int(&bb, "n", 1); + bson_from_buffer(&b, &bb); + options = MONGO_INDEX_UNIQUE; + success = mongo_create_index(gfs->client, gfs->chunks_ns, &b, options, &out); + bson_destroy(&b); + if (!success) { + free((char*)gfs->dbname); + free((char*)gfs->prefix); + free((char*)gfs->files_ns); + free((char*)gfs->chunks_ns); + return FALSE; + } + + return TRUE; +} + +/*--------------------------------------------------------------------*/ + +void gridfs_destroy(gridfs* gfs) + +{ + if (gfs == NULL) return; + if (gfs->dbname) free((char*)gfs->dbname); + if (gfs->prefix) free((char*)gfs->prefix); + if (gfs->files_ns) free((char*)gfs->files_ns); + if (gfs->chunks_ns) free((char*)gfs->chunks_ns); +} + +/*--------------------------------------------------------------------*/ + +static bson gridfs_insert_file( gridfs* gfs, const char* name, + const bson_oid_t id, gridfs_offset length, + const char* contenttype) +{ + bson command; + bson res; + bson ret; + bson_buffer buf; + bson_iterator it; + + /* Check run md5 */ + bson_buffer_init(&buf); + bson_append_oid(&buf, "filemd5", &id); + bson_append_string(&buf, "root", gfs->prefix); + bson_from_buffer(&command, &buf); + assert(mongo_run_command(gfs->client, gfs->dbname, + &command, &res)); + bson_destroy(&command); + + /* Create and insert BSON for file metadata */ + bson_buffer_init(&buf); + bson_append_oid(&buf, "_id", &id); + if (name != NULL && *name != '\0') { + bson_append_string(&buf, "filename", name); + } + bson_append_long(&buf, "length", length); + bson_append_int(&buf, "chunkSize", DEFAULT_CHUNK_SIZE); + bson_append_date(&buf, "uploadDate", (bson_date_t)1000*time(NULL)); + bson_find(&it, &res, "md5"); + bson_append_string(&buf, "md5", bson_iterator_string(&it)); + bson_destroy(&res); + if (contenttype != NULL && *contenttype != '\0') { + bson_append_string(&buf, "contentType", contenttype); + } + bson_from_buffer(&ret, &buf); + mongo_insert(gfs->client, gfs->files_ns, &ret); + + return ret; +} + +/*--------------------------------------------------------------------*/ + +bson gridfs_store_buffer( gridfs* gfs, const char* data, + gridfs_offset length, const char* remotename, + const char * contenttype) + +{ + char const * const end = data + length; + bson_oid_t id; + int chunkNumber = 0; + int chunkLen; + bson * oChunk; + + /* Large files Assertion */ + assert(length <= 0xffffffff); + + /* Generate and append an oid*/ + bson_oid_gen(&id); + + /* Insert the file's data chunk by chunk */ + while (data < end) { + chunkLen = DEFAULT_CHUNK_SIZE < (unsigned int)(end - data) ? + DEFAULT_CHUNK_SIZE : (unsigned int)(end - data); + oChunk = chunk_new( id, chunkNumber, data, chunkLen ); + mongo_insert(gfs->client, gfs->chunks_ns, oChunk); + chunk_free(oChunk); + chunkNumber++; + data += chunkLen; + } + + /* Inserts file's metadata */ + return gridfs_insert_file(gfs, remotename, id, length, contenttype); +} + +/*--------------------------------------------------------------------*/ + +void gridfile_writer_init( gridfile* gfile, gridfs* gfs, + const char* remote_name, const char* content_type ) +{ + gfile->gfs = gfs; + + bson_oid_gen( &(gfile->id) ); + gfile->chunk_num = 0; + gfile->length = 0; + gfile->pending_len = 0; + gfile->pending_data = NULL; + + gfile->remote_name = (const char *)bson_malloc( strlen( remote_name ) + 1 ); + strcpy( (char *)gfile->remote_name, remote_name ); + + gfile->content_type = (const char *)bson_malloc( strlen( content_type ) ); + strcpy( (char *)gfile->content_type, content_type ); +} + +/*--------------------------------------------------------------------*/ + +void gridfile_write_buffer( gridfile* gfile, const char* data, gridfs_offset length ) +{ + + int bytes_left = 0; + int data_partial_len = 0; + int chunks_to_write = 0; + char* buffer; + bson* oChunk; + gridfs_offset to_write = length + gfile->pending_len; + + if ( to_write < DEFAULT_CHUNK_SIZE ) { /* Less than one chunk to write */ + if( gfile->pending_data ) { + gfile->pending_data = (char *)bson_realloc((void *)gfile->pending_data, gfile->pending_len + to_write); + memcpy( gfile->pending_data + gfile->pending_len, data, length ); + } else if (to_write > 0) { + gfile->pending_data = (char *)bson_malloc(to_write); + memcpy( gfile->pending_data, data, length ); + } + gfile->pending_len += length; + + } else { /* At least one chunk of data to write */ + + /* If there's a pending chunk to be written, we need to combine + * the buffer provided up to DEFAULT_CHUNK_SIZE. + */ + if ( gfile->pending_len > 0 ) { + chunks_to_write = to_write / DEFAULT_CHUNK_SIZE; + bytes_left = to_write % DEFAULT_CHUNK_SIZE; + + data_partial_len = DEFAULT_CHUNK_SIZE - gfile->pending_len; + buffer = (char *)bson_malloc( DEFAULT_CHUNK_SIZE ); + memcpy(buffer, gfile->pending_data, gfile->pending_len); + memcpy(buffer + gfile->pending_len, data, data_partial_len); + + oChunk = chunk_new(gfile->id, gfile->chunk_num, buffer, DEFAULT_CHUNK_SIZE); + mongo_insert(gfile->gfs->client, gfile->gfs->chunks_ns, oChunk); + chunk_free(oChunk); + gfile->chunk_num++; + gfile->length += DEFAULT_CHUNK_SIZE; + data += data_partial_len; + + chunks_to_write--; + + free(buffer); + } + + while( chunks_to_write > 0 ) { + oChunk = chunk_new(gfile->id, gfile->chunk_num, data, DEFAULT_CHUNK_SIZE); + mongo_insert(gfile->gfs->client, gfile->gfs->chunks_ns, oChunk); + chunk_free(oChunk); + gfile->chunk_num++; + chunks_to_write--; + gfile->length += DEFAULT_CHUNK_SIZE; + data += DEFAULT_CHUNK_SIZE; + } + + free(gfile->pending_data); + + /* If there are any leftover bytes, store them as pending data. */ + if( bytes_left == 0 ) + gfile->pending_data = NULL; + else { + gfile->pending_data = (char *)bson_malloc( bytes_left ); + memcpy( gfile->pending_data, data, bytes_left ); + } + + gfile->pending_len = bytes_left; + } +} + +/*--------------------------------------------------------------------*/ + +bson gridfile_writer_done( gridfile* gfile ) +{ + + /* write any remaining pending chunk data. + * pending data will always take up less than one chunk + */ + bson* oChunk; + if( gfile->pending_data ) + { + oChunk = chunk_new(gfile->id, gfile->chunk_num, gfile->pending_data, gfile->pending_len); + mongo_insert(gfile->gfs->client, gfile->gfs->chunks_ns, oChunk); + chunk_free(oChunk); + free(gfile->pending_data); + gfile->length += gfile->pending_len; + } + + /* insert into files collection */ + return gridfs_insert_file(gfile->gfs, gfile->remote_name, gfile->id, + gfile->length, gfile->content_type); +} + +/*--------------------------------------------------------------------*/ + +bson gridfs_store_file(gridfs* gfs, const char* filename, + const char* remotename, const char* contenttype) +{ + char buffer[DEFAULT_CHUNK_SIZE]; + FILE * fd; + bson_oid_t id; + int chunkNumber = 0; + gridfs_offset length = 0; + gridfs_offset chunkLen = 0; + bson* oChunk; + + /* Open the file and the correct stream */ + if (strcmp(filename, "-") == 0) fd = stdin; + else fd = fopen(filename, "rb"); + assert(fd != NULL); /* No such file */ + + /* Generate and append an oid*/ + bson_oid_gen(&id); + + /* Insert the file chunk by chunk */ + chunkLen = fread(buffer, 1, DEFAULT_CHUNK_SIZE, fd); + do { + oChunk = chunk_new( id, chunkNumber, buffer, chunkLen ); + mongo_insert(gfs->client, gfs->chunks_ns, oChunk); + chunk_free(oChunk); + length += chunkLen; + chunkNumber++; + chunkLen = fread(buffer, 1, DEFAULT_CHUNK_SIZE, fd); + } while (chunkLen != 0); + + /* Close the file stream */ + if (fd != stdin) fclose(fd); + + /* Large files Assertion */ + /* assert(length <= 0xffffffff); */ + + /* Optional Remote Name */ + if (remotename == NULL || *remotename == '\0') { + remotename = filename; } + + /* Inserts file's metadata */ + return gridfs_insert_file(gfs, remotename, id, length, contenttype); +} + +/*--------------------------------------------------------------------*/ + +void gridfs_remove_filename(gridfs* gfs, const char* filename ) + +{ + bson query; + bson_buffer buf; + mongo_cursor* files; + bson file; + bson_iterator it; + bson_oid_t id; + bson b; + + bson_buffer_init(&buf); + bson_append_string(&buf, "filename", filename); + bson_from_buffer(&query, &buf); + files = mongo_find(gfs->client, gfs->files_ns, &query, NULL, 0, 0, 0); + bson_destroy(&query); + + /* Remove each file and it's chunks from files named filename */ + while (mongo_cursor_next(files)) { + file = files->current; + bson_find(&it, &file, "_id"); + id = *bson_iterator_oid(&it); + + /* Remove the file with the specified id */ + bson_buffer_init(&buf); + bson_append_oid(&buf, "_id", &id); + bson_from_buffer(&b, &buf); + mongo_remove( gfs->client, gfs->files_ns, &b); + bson_destroy(&b); + + /* Remove all chunks from the file with the specified id */ + bson_buffer_init(&buf); + bson_append_oid(&buf, "files_id", &id); + bson_from_buffer(&b, &buf); + mongo_remove( gfs->client, gfs->chunks_ns, &b); + bson_destroy(&b); + } + +} + +/*--------------------------------------------------------------------*/ + +int gridfs_find_query(gridfs* gfs, bson* query, + gridfile* gfile ) + +{ + bson_buffer date_buffer; + bson uploadDate; + bson_buffer buf; + bson finalQuery; + bson out; + int i; + + bson_buffer_init(&date_buffer); + bson_append_int(&date_buffer, "uploadDate", -1); + bson_from_buffer(&uploadDate, &date_buffer); + bson_buffer_init(&buf); + bson_append_bson(&buf, "query", query); + bson_append_bson(&buf, "orderby", &uploadDate); + bson_from_buffer(&finalQuery, &buf); + + + i = (mongo_find_one(gfs->client, gfs->files_ns, + &finalQuery, NULL, &out)); + bson_destroy(&uploadDate); + bson_destroy(&finalQuery); + if (!i) + return FALSE; + else { + gridfile_init(gfs, &out, gfile); + bson_destroy(&out); + return TRUE; + } +} + +/*--------------------------------------------------------------------*/ + +int gridfs_find_filename(gridfs* gfs, const char* filename, + gridfile* gfile) + +{ + bson query; + bson_buffer buf; + int i; + + bson_buffer_init(&buf); + bson_append_string(&buf, "filename", filename); + bson_from_buffer(&query, &buf) ; + i = gridfs_find_query(gfs, &query, gfile); + bson_destroy(&query); + return i; +} + +/*--------------------------------------------------------------------*/ + +int gridfile_init(gridfs* gfs, bson* meta, gridfile* gfile) + +{ + gfile->gfs = gfs; + gfile->pos = 0; + gfile->meta = (bson*)bson_malloc(sizeof(bson)); + if (gfile->meta == NULL) return FALSE; + bson_copy(gfile->meta, meta); + return TRUE; +} + +/*--------------------------------------------------------------------*/ + +void gridfile_destroy(gridfile* gfile) + +{ + bson_destroy(gfile->meta); + free(gfile->meta); +} + +/*--------------------------------------------------------------------*/ + +bson_bool_t gridfile_exists(gridfile* gfile) + +{ + return (bson_bool_t)(gfile != NULL || gfile->meta == NULL); +} + +/*--------------------------------------------------------------------*/ + +const char* gridfile_get_filename(gridfile* gfile) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, "filename"); + return bson_iterator_string(&it); +} + +/*--------------------------------------------------------------------*/ + +int gridfile_get_chunksize(gridfile* gfile) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, "chunkSize"); + return bson_iterator_int(&it); +} + +/*--------------------------------------------------------------------*/ + +gridfs_offset gridfile_get_contentlength(gridfile* gfile) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, "length"); + + if( bson_iterator_type( &it ) == bson_int ) + return (gridfs_offset)bson_iterator_int( &it ); + else + return (gridfs_offset)bson_iterator_long( &it ); +} + +/*--------------------------------------------------------------------*/ + +const char *gridfile_get_contenttype(gridfile* gfile) + +{ + bson_iterator it; + + if (bson_find(&it, gfile->meta, "contentType")) + return bson_iterator_string( &it ); + else return NULL; +} + +/*--------------------------------------------------------------------*/ + +bson_date_t gridfile_get_uploaddate(gridfile* gfile) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, "uploadDate"); + return bson_iterator_date( &it ); +} + +/*--------------------------------------------------------------------*/ + +const char* gridfile_get_md5(gridfile* gfile) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, "md5"); + return bson_iterator_string( &it ); +} + +/*--------------------------------------------------------------------*/ + +const char* gridfile_get_field(gridfile* gfile, const char* name) + +{ + bson_iterator it; + + bson_find(&it, gfile->meta, name); + return bson_iterator_value( &it ); +} + +/*--------------------------------------------------------------------*/ + +bson_bool_t gridfile_get_boolean(gridfile* gfile, const char* name) +{ + bson_iterator it; + + bson_find(&it, gfile->meta, name); + return bson_iterator_bool( &it ); +} + +/*--------------------------------------------------------------------*/ +bson gridfile_get_metadata(gridfile* gfile) + +{ + bson sub; + bson_iterator it; + + if (bson_find(&it, gfile->meta, "metadata")) { + bson_iterator_subobject( &it, &sub ); + return sub; + } + else { + bson_empty(&sub); + return sub; + } +} + +/*--------------------------------------------------------------------*/ + +int gridfile_get_numchunks(gridfile* gfile) + +{ + bson_iterator it; + gridfs_offset length; + gridfs_offset chunkSize; + double numchunks; + + bson_find(&it, gfile->meta, "length"); + + if( bson_iterator_type( &it ) == bson_int ) + length = (gridfs_offset)bson_iterator_int( &it ); + else + length = (gridfs_offset)bson_iterator_long( &it ); + + bson_find(&it, gfile->meta, "chunkSize"); + chunkSize = bson_iterator_int(&it); + numchunks = ((double)length/(double)chunkSize); + return (numchunks - (int)numchunks > 0) + ? (int)(numchunks+1) + : (int)(numchunks); +} + +/*--------------------------------------------------------------------*/ + +bson gridfile_get_chunk(gridfile* gfile, int n) + +{ + bson query; + bson out; + bson_buffer buf; + bson_iterator it; + bson_oid_t id; + + bson_buffer_init(&buf); + bson_find(&it, gfile->meta, "_id"); + id = *bson_iterator_oid(&it); + bson_append_oid(&buf, "files_id", &id); + bson_append_int(&buf, "n", n); + bson_from_buffer(&query, &buf); + + assert(mongo_find_one(gfile->gfs->client, + gfile->gfs->chunks_ns, + &query, NULL, &out)); + return out; +} + +/*--------------------------------------------------------------------*/ + +mongo_cursor* gridfile_get_chunks(gridfile* gfile, int start, int size) + +{ + bson_iterator it; + bson_oid_t id; + bson_buffer gte_buf; + bson gte_bson; + bson_buffer query_buf; + bson query_bson; + bson_buffer orderby_buf; + bson orderby_bson; + bson_buffer command_buf; + bson command_bson; + + bson_find(&it, gfile->meta, "_id"); + id = *bson_iterator_oid(&it); + + bson_buffer_init(&query_buf); + bson_append_oid(&query_buf, "files_id", &id); + if (size == 1) { + bson_append_int(&query_buf, "n", start); + } else { + bson_buffer_init(>e_buf); + bson_append_int(>e_buf, "$gte", start); + bson_from_buffer(>e_bson, >e_buf); + bson_append_bson(&query_buf, "n", >e_bson); + } + bson_from_buffer(&query_bson, &query_buf); + + bson_buffer_init(&orderby_buf); + bson_append_int(&orderby_buf, "n", 1); + bson_from_buffer(&orderby_bson, &orderby_buf); + + bson_buffer_init(&command_buf); + bson_append_bson(&command_buf, "query", &query_bson); + bson_append_bson(&command_buf, "orderby", &orderby_bson); + bson_from_buffer(&command_bson, &command_buf); + + return mongo_find(gfile->gfs->client, gfile->gfs->chunks_ns, + &command_bson, NULL, size, 0, 0); +} + +/*--------------------------------------------------------------------*/ + +gridfs_offset gridfile_write_file(gridfile* gfile, FILE *stream) + +{ + int i; + size_t len; + bson chunk; + bson_iterator it; + const char* data; + const int num = gridfile_get_numchunks( gfile ); + int xx; + + for ( i=0; i<num; i++ ){ + chunk = gridfile_get_chunk( gfile, i ); + bson_find( &it, &chunk, "data" ); + len = bson_iterator_bin_len( &it ); + data = bson_iterator_bin_data( &it ); + xx = fwrite( data , sizeof(char), len, stream ); + } + + return gridfile_get_contentlength(gfile); +} + +/*--------------------------------------------------------------------*/ + +gridfs_offset gridfile_read(gridfile* gfile, gridfs_offset size, char* buf) + +{ + mongo_cursor* chunks; + bson chunk; + + int first_chunk; + int last_chunk; + int total_chunks; + gridfs_offset chunksize; + gridfs_offset contentlength; + gridfs_offset bytes_left; + int i; + bson_iterator it; + gridfs_offset chunk_len; + const char * chunk_data; + + contentlength = gridfile_get_contentlength(gfile); + chunksize = gridfile_get_chunksize(gfile); + size = (contentlength - gfile->pos < size) + ? contentlength - gfile->pos + : size; + bytes_left = size; + + first_chunk = (gfile->pos)/chunksize; + last_chunk = (gfile->pos+size-1)/chunksize; + total_chunks = last_chunk - first_chunk + 1; + chunks = gridfile_get_chunks(gfile, first_chunk, total_chunks); + + for (i = 0; i < total_chunks; i++) { + mongo_cursor_next(chunks); + chunk = chunks->current; + bson_find(&it, &chunk, "data"); + chunk_len = bson_iterator_bin_len( &it ); + chunk_data = bson_iterator_bin_data( &it ); + if (i == 0) { + chunk_data += (gfile->pos)%chunksize; + chunk_len -= (gfile->pos)%chunksize; + } + if (bytes_left > chunk_len) { + memcpy(buf, chunk_data, chunk_len); + bytes_left -= chunk_len; + buf += chunk_len; + } else { + memcpy(buf, chunk_data, bytes_left); + } + } + + mongo_cursor_destroy(chunks); + gfile->pos = gfile->pos + size; + + return size; +} + +/*--------------------------------------------------------------------*/ + +gridfs_offset gridfile_seek(gridfile* gfile, gridfs_offset offset) + +{ + gridfs_offset length; + + length = gridfile_get_contentlength(gfile); + gfile->pos = length < offset ? length : offset; + return gfile->pos; +} + +/*==============================================================*/ /* --- mongo_except.h */ /* This file is based loosely on cexcept (http://www.nicemice.net/cexcept/). I @@ . ______________________________________________________________________ RPM Package Manager http://rpm5.org CVS Sources Repository [email protected]
